#!/usr/bin/python
# -*- coding: utf-8 -*-
########################################################################
## Read wiki file as parameter and write a new file. Replace any
## SEARCH macro with a corresponding DBQUERY macro if possible.
##
## Currently supported are only query searches and the most common
## search parameters. If an unsupported parameter is detected, the
## search is left untouched and a corresponding message is logged.
##
## Search parameters are left untouched and should work in most cases.
## Beware that DBQUERY supports the ~ operator only as workaround.
## For example Foobar~'Topic*' will match for FirstTopic123, too.
## For a perfect match, ^ and $ need to be added to each RHS.
## This script does _not_ do that.
##
## Further modifications:
## - Table headers are modified and centered: |*cell*| -> |  *cell*  |
## - Something like the following is rewritten:
##   Before: $percntFORMFIELD{"Foo" topic="$formfield(Bar)"}$percnt
##   After:  $expand(@Bar.Foo)
##
## The script should be used with caution and conversions should be
## verified by checking the messages and comparing some results before
## deactivating the testmode. Comment/uncomment log messages as needed.
##
## This script requires Python 2.7.
##
## Stefan Hamacher, zak256@mail.de, 2018-03-28, Version 1.0
##
########################################################################
## Requirements
import os
import sys
import re
import logging
from collections import OrderedDict

########################################################################
## Configuration and parameters

# If set to False, move new file to actual file and overwrite the latter.
testmode = True

# Create backup file with extra suffix before overwriting file.
# Only relevant with testmode=False.
do_backup = True

# If the main topic file has changed, the timestamp of the
# corresponding rcs file should be updated as well.
touch_rcs = True

# Suffixes for files
suffix_newfile = "-dbquery"
suffix_backupfile = "-searchbackup"

# Include topic names to ignore in this list
blacklist = []

# Read parameter
try:
    filename = sys.argv[1]
except IndexError:
    print "Usage: {} <wikifile>".format(__file__)
    sys.exit(1)

# Regex for SEARCH macro
pattern_search = re.compile(
    r'''(?P<pre>.*)
    %SEARCH\{\s*
    "(?P<query>.+?)"
    (?:\s+scope="(?P<scope>.+?)"
    |\s+web="(?P<web>.+?)"
    |\s+topic="(?P<topic>.+?)"
    |\s+excludetopic="(?P<excludetopic>.+?)"
    |\s+type="(?P<type>.+?)"
    |\s+casesensitive="(?P<casesensitive>.+?)"
    |\s+nototal="(?P<nototal>.+?)"
    |\s+zeroresults="(?P<zeroresults>.+?)"
    |\s+noempty="(?P<noempty>.+?)"
    |\s+nosearch="(?P<nosearch>.+?)"
    |\s+noheader="(?P<noheader>.+?)"
    |\s+nosummary="(?P<nosummary>.+?)"
    |\s+multiple="(?P<multiple>.+?)"
    |\s+separator="(?P<separator>.+?)"
    |\s+nonoise="(?P<nonoise>.+?)"
    |\s+order="(?P<order>.+?)"
    |\s+reverse="(?P<reverse>.+?)"
    |\s+date="(?P<date>.+?)"
    |\s+limit="(?P<limit>.+?)"
    |\s+header="(?P<header>.+?)"
    |\s+footer="(?P<footer>.+?)"
    |\s+format="(?P<format>.+?)")+
    \s*\}%
    (?P<post>.*)''', re.X + re.S)

# Regex for formfield
pattern_formfield = re.compile('^formfield\(([^\)]+)\)$')

# Logging
logging.basicConfig(format = "{} : %(levelname)s : %(message)s".format(filename), level = logging.DEBUG)
log = logging.getLogger()

########################################################################
## Functions
def touch(fname, times = None):
    """Update timestamp of `fname`."""
    if os.path.isfile(fname):
        with open(fname, 'a'):
            os.utime(fname, times)

def convert_search2dbquery(searchstring):
    """Try matching search pattern on `searchstring`. Return new string
    with dbquery macro if possible. Return None if match or conversion
    is not possible."""
    # Match line
    match_search = pattern_search.match(searchstring)
    if not match_search:
        log.error("Could not match line.")
        return None
    if match_search.group('type') != "query":
        log.error("UNSUPPORTED: Skipping non-query search ({}).".format(match_search.group('type')))
        return None

    # Prepare values
    resultparams = OrderedDict()

    query = match_search.group('query').strip()
    if not query:
        log.error("Could not get query.")
        return None
    topic = match_search.group('topic')
    if not topic:
        log.error("Could not get topic.")
        return None
    if "," in topic:
        topic = "(" + "|".join([ t for t in map(str.strip, topic.split(",")) ]) + ")"
    query_topic = "topic~'^{}$'".format(topic)
    if query == "1":
        resultparams["query"] = query_topic
    else:
        # Replace NOT -> !
        if re.search('(?:[^A-Za-z0-9]|^)NOT ', query):
            #log.debug("Found NOT in query: " + query)
            query = re.sub('([^A-Za-z0-9]|^)NOT ', r'\1! ', query)
            #log.debug("New query:          " + query)
        # Check if additional brackets are required
        match_or = re.match('(.+)\s+OR\s+', query)
        if match_or and match_or.group(1).count('(') - match_or.group(1).count(')') == 0:
            #log.debug("Added brackets around: " + query)
            query = "({})".format(query)
        #else:
        #    log.debug("Did not add brackets around: " + query)
        resultparams["query"] = "{} AND {}".format(query_topic, query)

    excludetopic = match_search.group('excludetopic')
    if excludetopic:
        log.error('UNSUPPORTED: Found excludetopic="' + excludetopic + '"')
        # TODO
        return None

    date_ = match_search.group('date')
    if date_:
        log.error('UNSUPPORTED: Found date="' + date_ + '"')
        # TODO
        return None

    multiple = match_search.group('multiple')
    if multiple:
        log.error('UNSUPPORTED: Found multiple="' + multiple + '"')
        # TODO
        return None

    # nonoise="on": shorthand for nosummary="on" nosearch="on" nototal="on" zeroresults="off" noheader="on" noempty="on"
    nosummary = False
    nosearch = False
    nototal = False
    zeroresults = True
    #TODO:noheader = 'Topics: Changed: By:'
    #TODO:noempty = False

    nonoise = match_search.group('nonoise')
    if nonoise and nonoise != "off":
        if nonoise == "on":
            nosummary = True
            nosearch = True
            nototal = True
            zeroresults = False
            #TODO:noheader = True
            #TODO:noempty = True
        else:
            log.error('Found nonoise="' + nonoise + '"')
            return None

    nosummaryParam = match_search.group('nosummary')
    if nosummaryParam:
        if nosummaryParam == "on":
            nosummary = True
        elif nosummaryParam == "off":
            nosummary = False
        else:
            log.error('Found nosummary="' + nosummaryParam + '"')
            return None

    nosearchParam = match_search.group('nosearch')
    if nosearchParam:
        if nosearchParam == "on":
            nosearch = True
        elif nosearchParam == "off":
            nosearch = False
        else:
            log.error('Found nosearch="' + nosearchParam + '"')
            return None

    nototalParam = match_search.group('nototal')
    if nototalParam:
        if nototalParam == "on":
            nototal = True
        elif nototalParam == "off":
            nototal = False
        else:
            log.error('Found nototal="' + nototal + '"')

    zeroresultsParam = match_search.group('zeroresults')
    if zeroresultsParam:
        if zeroresultsParam == "off" or zeroresults == "":
            zeroresults = False
        elif zeroresultsParam == "on":
            zeroresults = True
        else:
            log.error('UNSUPPORTED: Found zeroresults="' + zeroresultsParam + '"')
            return None

    noheaderParam = match_search.group('noheader')
    if noheaderParam:
        log.error('UNSUPPORTED: Found noheader="' + noheaderParam + '"')
        # TODO
        return None

    noemptyParam = match_search.group('noempty')
    if noemptyParam:
        log.error('UNSUPPORTED: Found noempty="' + noemptyParam + '"')
        # TODO
        return None

    footer = match_search.group('footer')
    if footer:
        log.debug('Found footer="' + footer + '"')
        resultparams["footer"] = '$n' + footer

    web = match_search.group('web')
    if not web:
        log.error("Could not get web.")
        return None
    if "," in web:
        log.debug('Detected search with multiple webs: ' + web)
        resultparams["webs"] = web
    else:
        resultparams["web"] = web

    sort = match_search.group('order')
    if sort:
        match_sort = pattern_formfield.match(sort)
        if match_sort:
            sort = match_sort.group(1)
            resultparams["sort"] = sort
        elif sort == "topic":
            resultparams["sort"] = "name"
        elif sort == "modified":
            resultparams["sort"] = "modified"  # "info.date"
        else:
            log.error("UNSUPPORTED: Could not process order: " + sort)
            return None

    separator = match_search.group('separator')
    if separator:
        if separator == '$n':
            log.debug('Ignoring default separator="' + separator + '"')
        else:
            log.debug('Found separator="' + separator + '"')
            resultparams["separator"] = separator

    reverse = match_search.group('reverse')
    if reverse and reverse != "off":
        #log.debug('Found reverse="' + reverse + '"')
        if reverse == "on":
            resultparams["reverse"] = "on"
        else:
            log.error('Unknown value reverse="' + reverse + '"')
            return None

    limit = match_search.group('limit')
    if limit:
        log.debug('Set limit="' + limit + '"')
        resultparams["limit"] = limit

    header = match_search.group('header')
    if header:
        resultparams["header"] = header + '$n'
    #else:
    #    log.debug("No header.")

    format_ = match_search.group('format')
    if not format_:
        log.error("Could not get format.")
        return None
    format_ = format_.rstrip()
    format_ = format_.rstrip('\\')
    format_ = re.sub('\$perce?ntFORMFIELD\{\\\\"(\w+)\\\\" topic=\\\\"\$formfield\((\w+)\)\\\\"\}\$perce?nt', r'$expand(@\2.\1)', format_)
    format_ = re.sub('!?\$wikiname', r'$expand(info.author)', format_)
    format_ = format_.replace('$parent', '[[$expand(parent)]]')
    #format_ = format_.replace('$isodate', '$percntDATETIME{\\"epoch $expand(info.date)\\"}$percnt')
    format_ = format_.replace('$isodate', '$formatTime(info.date, \'$year-$mo-$day $hour:$minute\')')
    if '$createdate' in format_:
        log.warning("Found unsupported '$createdate'.")
    resultparams["format"] = format_

    # Special formatting
    if not nosummary:
        log.error("UNSUPPORTED: summary is activated.")

    if not nosearch:
        log.error("UNSUPPORTED: Cannot set search query.")
    #    if "header" in resultparams:
    #        resultparams["header"] = "Searched: *TODO* Number of topics: *$count*$n" + resultparams["header"]
    #    else:
    #        resultparams["header"] = "Searched: *TODO* Number of topics: *$count*$n"

    if not nototal:
        resultparams["footer"] = '$n' + "*Number of topics: $count*" + resultparams.get("footer", "")

    if not zeroresults:
        resultparams["hidenull"] = "on"

    if '|' in resultparams.get("format", ""):
        # Probably wiki table syntax detected. Newlines need to be replaced. See https://foswiki.org/Tasks/Item14661
        resultparams["newline"] = '<br>'

    # Center header cells
    if "header" in resultparams:
        resultparams["header"] = re.sub('\|\s*\*', '|  *', resultparams["header"])
        resultparams["header"] = re.sub('\*\s*\|', '*  |', resultparams["header"])

    # Construct result line
    paramstrings = [ '"{}"'.format(resultparams["query"]) ]
    paramstrings += [ '{}="{}"'.format(k, v) for k, v in resultparams.iteritems() if k not in [ "query", "header", "format", "footer" ] ]
    headerstr = '\nheader="{}"'.format(resultparams["header"]) if "header" in resultparams else ""
    footerstr = '\nfooter="{}"'.format(resultparams["footer"]) if "footer" in resultparams else ""
    formatstr = '\nformat="{}"'.format(resultparams["format"]) if "format" in resultparams else ""
    lb = '\n' if '\n' in resultparams.get("format", "") else ''  # add cosmetic linebreak after multi line format string
    line_dbquery = '%DBQUERY{' + " ".join(paramstrings) + headerstr + footerstr + formatstr + lb + '}%'

    pre = match_search.group('pre')
    if pre:
        log.info("Found pre text: " + pre)
        line_dbquery = pre + line_dbquery
    post = match_search.group('post')
    if post:
        log.info("Found post text: " + post)
        line_dbquery += post

    return line_dbquery

########################################################################
## Main

# Ignore files in blacklist
if os.path.splitext(os.path.basename(filename))[0] in blacklist:
    #log.debug("Ignoring blacklisted file.")
    sys.exit(0)

# grep for SEARCH to check if file needs to be processed at all
try:
    with open(filename, 'r') as fh:
        if not '%SEARCH{' in fh.read():
            log.info("No SEARCH found.")
            sys.exit(0)
except IOError as e:
    log.error("Could not read file: {}".format(e.strerror))
    sys.exit(1)

# Process file with at least one SEARCH
searchline = ""
didsomething = False
newlines = []
try:
    with open(filename, 'r') as fh:
        in_search = False
        brackets = 0
        for line in fh:
            if line.startswith("%SEARCH{"):
                if line.count("%SEARCH{") > 1:
                    log.error("Detected unsupported search within another search. Aborting.")
                    sys.exit(1)
                in_search = True
            if in_search:
                brackets += line.count("{")
                brackets -= line.count("}")
                searchline += line.rstrip() + "\n"
            else:
                newlines.append(line.rstrip())
            if "}%" in line and searchline and brackets == 0:
                in_search = False
                searchline = searchline.strip()
                dbqueryline = convert_search2dbquery(searchline)
                if dbqueryline:
                    newlines.append(dbqueryline)
                    didsomething = True
                else:
                    newlines.append(searchline)
                searchline = ""
except IOError as e:
    log.error("Could not read file: {}".format(e.strerror))
    sys.exit(1)

if didsomething:
    # Write new file
    base, ext = os.path.splitext(filename)
    newfile = base + suffix_newfile + ext
    backupfile = base + suffix_backupfile + ext
    with open(newfile, 'w') as fh:
        for l in newlines:
            fh.write(l + '\n')
    if not testmode:
        if do_backup:
            os.rename(filename, backupfile)
        os.rename(newfile, filename)
        log.info("File updated.")
    else:
        log.info("File created: " + newfile)
    if touch_rcs and not testmode:
        touch(filename + ",v")