Warning, /sdk/pology/bin/poascribe is written in an unsupported language. File is not indexed.

0001 #!/usr/bin/env python3
0002 # -*- coding: UTF-8 -*-
0003 
0004 try:
0005     import fallback_import_paths
0006 except:
0007     pass
0008 
0009 import datetime
0010 import locale
0011 import os
0012 import re
0013 import sys
0014 from tempfile import NamedTemporaryFile
0015 import time
0016 
0017 from pology import PologyError, version, _, n_, t_
0018 from pology.ascript import collect_ascription_associations
0019 from pology.ascript import collect_ascription_history
0020 from pology.ascript import collect_ascription_history_segment
0021 from pology.ascript import ascription_equal, merge_modified
0022 from pology.ascript import ascribe_modification, ascribe_review
0023 from pology.ascript import first_non_fuzzy, has_tracked_parts
0024 from pology.ascript import make_ascription_selector
0025 from pology.ascript import AscPoint
0026 from pology.catalog import Catalog
0027 from pology.header import Header, TZInfo, format_datetime
0028 from pology.message import Message, MessageUnsafe
0029 from pology.gtxtools import msgfmt
0030 from pology.colors import ColorOptionParser, cjoin
0031 import pology.config as pology_config
0032 from pology.diff import msg_ediff, msg_ediff_to_new
0033 from pology.diff import editprob
0034 from pology.fsops import str_to_unicode, unicode_to_str
0035 from pology.fsops import collect_paths_cmdline, collect_catalogs
0036 from pology.fsops import mkdirpath, join_ncwd
0037 from pology.fsops import exit_on_exception
0038 from pology.getfunc import get_hook_ireq
0039 from pology.merge import merge_pofile
0040 from pology.monitored import Monlist
0041 from pology.msgreport import warning_on_msg, report_msg_content
0042 from pology.msgreport import report_msg_to_lokalize
0043 from pology.report import report, error, format_item_list
0044 from pology.report import init_file_progress
0045 from pology.stdcmdopt import add_cmdopt_incexc, add_cmdopt_filesfrom
0046 from pology.tabulate import tabulate
0047 
0048 
0049 # Wrapping in ascription catalogs.
0050 _ascwrapping = ["fine"]
0051 
0052 # Flag used to mark diffed messages.
0053 # NOTE: All diff flags should start with 'ediff', as some other scripts
0054 # only need to check if any of them is present.
0055 _diffflag = "ediff"
0056 _diffflag_tot = "ediff-total"
0057 _diffflag_ign = "ediff-ignored"
0058 
0059 # Flags used to explicitly mark messages as reviewed or unreviewed.
0060 _revdflags = ("reviewed", "revd", "rev") # synonyms
0061 _urevdflags = ("unreviewed", "nrevd", "nrev") # synonyms
0062 
0063 # Comment used to show ascription chain in messages marked for review.
0064 _achncmnt = "~ascto:"
0065 
0066 # String used to separate tags to review flags.
0067 _flagtagsep = "/"
0068 
0069 _diffflags = (_diffflag, _diffflag_tot, _diffflag_ign)
0070 _all_flags = _diffflags + _revdflags + _urevdflags
0071 _all_flags = sorted(_all_flags, key=lambda x: (-len(x), x))
0072 # ...this order is necessary for proper |-linking in regexes.
0073 _all_cmnts = (_achncmnt,)
0074 
0075 # Datetime at the moment the script is started.
0076 _dt_start = datetime.datetime(*(time.localtime()[:6] + (0, TZInfo())))
0077 
0078 
0079 def main ():
0080 
0081     locale.setlocale(locale.LC_ALL, "")
0082 
0083     mode_spec = (
0084         ("status", ("st",)),
0085         ("commit", ("co", "ci", "mo")),
0086         ("diff", ("di",)),
0087         ("purge", ("pu",)),
0088         ("history", ("hi",)),
0089     )
0090     mode_allnames = set()
0091     mode_tolong = {}
0092     for name, syns in mode_spec:
0093         mode_allnames.add(name)
0094         mode_allnames.update(syns)
0095         mode_tolong[name] = name
0096         mode_tolong.update((s, name) for s in syns)
0097 
0098     known_editors = {
0099         "lokalize": report_msg_to_lokalize,
0100     }
0101 
0102     # Setup options and parse the command line.
0103     usage = _("@info command usage",
0104         "%(cmd)s MODE [OPTIONS] [PATHS...]",
0105         cmd="%prog")
0106     desc = _("@info command description",
0107         "Keep track of who, when, and how, has translated, modified, "
0108         "or reviewed messages in a collection of PO files.")
0109     ver = _("@info command version",
0110         "%(cmd)s (Pology) %(version)s\n"
0111         "Copyright © 2008, 2009, 2010 "
0112         "Chusslove Illich (Часлав Илић) <%(email)s>",
0113         cmd="%prog", version=version(), email="caslav.ilic@gmx.net")
0114 
0115     opars = ColorOptionParser(usage=usage, description=desc, version=ver)
0116     opars.add_option(
0117         "-a", "--select-ascription",
0118         metavar=_("@info command line value placeholder", "SELECTOR[:ARGS]"),
0119         action="append", dest="aselectors", default=None,
0120         help=_("@info command line option description",
0121                "Select a message from ascription history by this selector. "
0122                "Can be repeated, in which case the message is selected "
0123                "if all selectors match it."))
0124     opars.add_option(
0125         "-A", "--min-adjsim-diff",
0126         metavar=_("@info command line value placeholder", "RATIO"),
0127         action="store", dest="min_adjsim_diff", default=None,
0128         help=_("@info command line option description",
0129                "Minimum adjusted similarity between two versions of a message "
0130                "needed to show the embedded difference. "
0131                "Range 0.0-1.0, where 0 means always to show the difference, "
0132                "and 1 never to show it; a convenient range is 0.6-0.8. "
0133                "When the difference is not shown, the '%(flag)s' flag is "
0134                "added to the message.",
0135                flag=_diffflag_ign))
0136     opars.add_option(
0137         "-b", "--show-by-file",
0138         action="store_true", dest="show_by_file", default=False,
0139         help=_("@info command line option description",
0140                "Next to global summary, also present results by file."))
0141     opars.add_option(
0142         "-C", "--no-vcs-commit",
0143         action="store_false", dest="vcs_commit", default=None,
0144         help=_("@info command line option description",
0145                "Do not commit catalogs to version control "
0146                "(when version control is used)."))
0147     opars.add_option(
0148         "-d", "--depth",
0149         metavar=_("@info command line value placeholder", "LEVEL"),
0150         action="store", dest="depth", default=None,
0151         help=_("@info command line option description",
0152                "Consider ascription history up to this level into the past."))
0153     opars.add_option(
0154         "-D", "--diff-reduce-history",
0155         metavar=_("@info command line value placeholder", "SPEC"),
0156         action="store", dest="diff_reduce_history", default=None,
0157         help=_("@info command line option description",
0158                "Reduce each message in history to a part of the difference "
0159                "from the first earlier modification: to added, removed, or "
0160                "equal segments. "
0161                "The value begins with one of the characters 'a', 'r', or 'e', "
0162                "followed by substring that will be used to separate "
0163                "selected difference segments in resulting messages "
0164                "(if this substring is empty, space is used)."))
0165     opars.add_option(
0166         "-F", "--filter",
0167         metavar=_("@info command line value placeholder", "NAME"),
0168         action="append", dest="filters", default=None,
0169         help=_("@info command line option description",
0170                "Pass relevant message text fields through a filter before "
0171                "matching or comparing them (relevant in some modes). "
0172                "Can be repeated to add several filters."))
0173     opars.add_option(
0174         "-G", "--show-filtered",
0175         action="store_true", dest="show_filtered", default=False,
0176         help=_("@info command line option description",
0177                "When operating under a filter, also show filtered versions "
0178                "of whatever is shown in original (e.g. in diffs)."))
0179     opars.add_option(
0180         "-k", "--keep-flags",
0181         action="store_true", dest="keep_flags", default=False,
0182         help=_("@info command line option description",
0183                "Do not remove review-significant flags from messages "
0184                "(possibly convert them as appropriate)."))
0185     opars.add_option(
0186         "-m", "--message",
0187         metavar=_("@info command line value placeholder", "TEXT"),
0188         action="store", dest="message", default=None,
0189         help=_("@info command line option description",
0190                "Version control commit message for original catalogs, "
0191                "when %(opt)s is in effect.",
0192                opt="-c"))
0193     opars.add_option(
0194         "-o", "--open-in-editor",
0195         metavar=("|".join(sorted(known_editors))),
0196         action="store", dest="po_editor", default=None,
0197         help=_("@info command line option description",
0198                "Open selected messages in one of the supported PO editors."))
0199     opars.add_option(
0200         "-L", "--max-fraction-select",
0201         metavar=_("@info command line value placeholder", "FRACTION"),
0202         action="store", dest="max_fraction_select", default=None,
0203         help=_("@info command line option description",
0204                "Select messages in a catalog only if the total number "
0205                "of selected messages in that catalog would be at most "
0206                "the given fraction (0.0-1.0) of total number of messages."))
0207     opars.add_option(
0208         "-s", "--selector",
0209         metavar=_("@info command line value placeholder", "SELECTOR[:ARGS]"),
0210         action="append", dest="selectors", default=None,
0211         help=_("@info command line option description",
0212                "Consider only messages matched by this selector. "
0213                "Can be repeated, in which case the message is selected "
0214                "if all selectors match it."))
0215     opars.add_option(
0216         "-t", "--tag",
0217         metavar=_("@info command line value placeholder", "TAG"),
0218         action="store", dest="tags", default=None,
0219         help=_("@info command line option description",
0220                "Tag to add or consider in ascription records. "
0221                "Several tags may be given separated by commas."))
0222     opars.add_option(
0223         "-u", "--user",
0224         metavar=_("@info command line value placeholder", "USER"),
0225         action="store", dest="user", default=None,
0226         help=_("@info command line option description",
0227                "User in whose name the operation is performed."))
0228     opars.add_option(
0229         "-U", "--update-headers",
0230         action="store_true", dest="update_headers", default=None,
0231         help=_("@info command line option description",
0232                "Update headers in catalogs which contain modifications "
0233                "before committing them, with user's translator information."))
0234     opars.add_option(
0235         "-v", "--verbose",
0236         action="store_true", dest="verbose", default=False,
0237         help=_("@info command line option description",
0238                "Output more detailed progress info."))
0239     opars.add_option(
0240         "-w", "--write-modified",
0241         metavar=_("@info command line value placeholder", "FILE"),
0242         action="store", dest="write_modified", default=None,
0243         help=_("@info command line option description",
0244                "Write paths of all original catalogs modified by "
0245                "ascription operations into the given file."))
0246     opars.add_option(
0247         "-x", "--externals",
0248         metavar=_("@info command line value placeholder", "PYFILE"),
0249         action="append", dest="externals", default=[],
0250         help=_("@info command line option description",
0251                "Collect optional functionality from an external Python file "
0252                "(selectors, etc)."))
0253     opars.add_option(
0254         "--all-reviewed",
0255         action="store_true", dest="all_reviewed", default=False,
0256         help=_("@info command line option description",
0257                "Ascribe all messages as reviewed on commit, "
0258                "overriding any existing review elements. "
0259                "Tags given by %(opt)s apply. "
0260                "This should not be done in day-to-day practice; "
0261                "the primary use is initial review ascription.",
0262                opt="--tag"))
0263     add_cmdopt_filesfrom(opars)
0264     add_cmdopt_incexc(opars)
0265 
0266     (options, free_args) = opars.parse_args(str_to_unicode(sys.argv[1:]))
0267 
0268     # Parse operation mode and its arguments.
0269     if len(free_args) < 1:
0270         error(_("@info", "Operation mode not given."))
0271     rawmodename = free_args.pop(0)
0272     modename = mode_tolong.get(rawmodename)
0273     if modename is None:
0274         flatmodes = ["/".join((x[0],) + x[1]) for x in mode_spec]
0275         error(_("@info",
0276                 "Unknown operation mode '%(mode)s' "
0277                 "(known modes: %(modelist)s).",
0278                 mode=rawmodename,
0279                 modelist=format_item_list(flatmodes)))
0280 
0281     # For options not issued, read values from user configuration.
0282     # Configuration values can also be issued by mode using
0283     # C{afield/amode = value} syntax, which takes precedence over
0284     # general fields (e.g. C{filters/review} vs. C{filters}).
0285     cfgsec = pology_config.section("poascribe")
0286     for optname, getvalf, defval in (
0287         ("aselectors", cfgsec.strdlist, []),
0288         ("vcs-commit", cfgsec.boolean, True),
0289         ("po-editor", cfgsec.string, None),
0290         ("filters", cfgsec.strslist, []),
0291         ("min-adjsim-diff", cfgsec.real, 0.0),
0292         ("selectors", cfgsec.strdlist, []),
0293         ("tags", cfgsec.string, ""),
0294         ("user", cfgsec.string, None),
0295         ("update-headers", cfgsec.boolean, False),
0296         ("diff-reduce-history", cfgsec.string, None),
0297         ("max-fraction-select", cfgsec.real, 1.01),
0298     ):
0299         uoptname = optname.replace("-", "_")
0300         if getattr(options, uoptname) is None:
0301             for fldname in ("%s/%s" % (optname, modename), optname):
0302                 fldval = getvalf(fldname, None)
0303                 if fldval is not None:
0304                     break
0305             if fldval is None:
0306                 fldval = defval
0307             setattr(options, uoptname, fldval)
0308 
0309     # Convert options to non-string types.
0310     def valconv_editor (edkey):
0311         msgrepf = known_editors.get(edkey)
0312         if msgrepf is None:
0313             error(_("@info",
0314                     "PO editor '%(ed)s' is not among "
0315                     "the supported editors: %(edlist)s.",
0316                     ed=edkey, edlist=format_item_list(sorted(known_editors))))
0317         return msgrepf
0318     def valconv_tags (cstr):
0319         return set(x.strip() for x in cstr.split(","))
0320     for optname, valconv in (
0321         ("max-fraction-select", float),
0322         ("min-adjsim-diff", float),
0323         ("po-editor", valconv_editor),
0324         ("tags", valconv_tags),
0325     ):
0326         uoptname = optname.replace("-", "_")
0327         valraw = getattr(options, uoptname, None)
0328         if valraw is not None:
0329             try:
0330                 value = valconv(valraw)
0331             except TypeError:
0332                 error(_("@info",
0333                         "Value '%(val)s' to option '%(opt)s' is of wrong type.",
0334                         val=valraw, opt=("--" + optname)))
0335             setattr(options, uoptname, value)
0336 
0337     # Collect any external functionality.
0338     for xmod_path in options.externals:
0339         collect_externals(xmod_path)
0340 
0341     # Create history filter if requested, store it in options.
0342     options.hfilter = None
0343     options.sfilter = None
0344     if options.filters:
0345         hfilters = []
0346         for hspec in options.filters:
0347             hfilters.append(get_hook_ireq(hspec, abort=True))
0348         def hfilter_composition (text):
0349             for hfilter in hfilters:
0350                 text = hfilter(text)
0351             return text
0352         options.hfilter = hfilter_composition
0353         if options.show_filtered:
0354             options.sfilter = options.hfilter
0355 
0356     # Create specification for reducing historical messages to diffs.
0357     options.addrem = None
0358     if options.diff_reduce_history:
0359         options.addrem = options.diff_reduce_history
0360         if options.addrem[:1] not in ("a", "e", "r"):
0361             error(_("@info",
0362                     "Value '%(val)s' to option '%(opt)s' must start "
0363                     "with '%(char1)s', '%(char2)s', or '%(char3)s'.",
0364                     val=options.addrem, opt="--diff-reduce-history",
0365                     char1="a", char2="e", char3="r"))
0366 
0367     # Create selectors if any explicitly given.
0368     selector = None
0369     if options.selectors:
0370         selector = make_ascription_selector(options.selectors)
0371     aselector = None
0372     if options.aselectors:
0373         aselector = make_ascription_selector(options.aselectors, hist=True)
0374 
0375     # Assemble operation mode.
0376     needuser = False
0377     canselect = False
0378     canaselect = False
0379     class _Mode: pass
0380     mode = _Mode()
0381     mode.name = modename
0382     if 0: pass
0383     elif mode.name == "status":
0384         mode.execute = status
0385         mode.selector = selector or make_ascription_selector(["any"])
0386         canselect = True
0387     elif mode.name == "commit":
0388         mode.execute = commit
0389         mode.selector = selector or make_ascription_selector(["any"])
0390         needuser = True
0391         canselect = True
0392     elif mode.name == "diff":
0393         mode.execute = diff
0394         mode.selector = selector or make_ascription_selector(["modar"])
0395         mode.aselector = aselector
0396         canselect = True
0397         canaselect = True
0398     elif mode.name == "purge":
0399         mode.execute = purge
0400         mode.selector = selector or make_ascription_selector(["any"])
0401         canselect = True
0402     elif mode.name == "history":
0403         mode.execute = history
0404         mode.selector = selector or make_ascription_selector(["any"])
0405         canselect = True
0406     else:
0407         error(_("@info",
0408                 "Unhandled operation mode '%(mode)s'.",
0409                 mode=mode.name))
0410 
0411     mode.user = None
0412     if needuser:
0413         if not options.user:
0414             error(_("@info",
0415                     "Operation mode '%(mode)s' requires a user "
0416                     "to be specified.",
0417                     mode=mode.name))
0418         mode.user = options.user
0419     if not canselect and selector:
0420         error(_("@info",
0421                 "Operation mode '%(mode)s' does not accept selectors.",
0422                 mode=mode.name))
0423     if not canaselect and aselector:
0424         error(_("@info",
0425                 "Operation mode '%(mode)s' does not accept history selectors.",
0426                 mode=mode.name))
0427 
0428     # Collect list of catalogs supplied through command line.
0429     # If none supplied, assume current working directory.
0430     catpaths = collect_paths_cmdline(rawpaths=free_args,
0431                                      incnames=options.include_names,
0432                                      incpaths=options.include_paths,
0433                                      excnames=options.exclude_names,
0434                                      excpaths=options.exclude_paths,
0435                                      filesfrom=options.files_from,
0436                                      elsecwd=True,
0437                                      respathf=collect_catalogs,
0438                                      abort=True)
0439 
0440     # Split catalogs into lists by ascription config,
0441     # and link them to their ascription catalogs.
0442     aconfs_catpaths = collect_ascription_associations(catpaths)
0443     assert_review_tags(aconfs_catpaths, options.tags)
0444 
0445     # Execute operation.
0446     mode.execute(options, aconfs_catpaths, mode)
0447 
0448     # Write out list of modified original catalogs if requested.
0449     if options.write_modified and _modified_cats:
0450         lfpath = options.write_modified
0451         f = open(lfpath, "w")
0452         f.write(("\n".join(sorted(_modified_cats)) + "\n").encode("utf-8"))
0453         f.close()
0454         report(_("@info",
0455                  "Paths of modified catalogs written to '%(file)s'.",
0456                  file=lfpath))
0457 
0458 
0459 def vcs_commit_catalogs (aconfs_catpaths, user, message=None, onabortf=None):
0460 
0461     report(_("@info:progress VCS is acronym for \"version control system\"",
0462              ">>>>> VCS is committing catalogs:"))
0463 
0464     # Attach paths to each distinct config, to commit them all at once.
0465     aconfs = []
0466     catpaths_byconf = {}
0467     for aconf, catpaths in aconfs_catpaths:
0468         if aconf not in catpaths_byconf:
0469             catpaths_byconf[aconf] = []
0470             aconfs.append(aconf)
0471         for catpath, acatpath in catpaths:
0472             catpaths_byconf[aconf].append(catpath)
0473             if os.path.isfile(acatpath):
0474                 catpaths_byconf[aconf].append(acatpath)
0475 
0476     # Commit by config.
0477     for aconf in aconfs:
0478         cmsg = message
0479         cmsgfile = None
0480         if not cmsg:
0481             cmsg = aconf.commitmsg
0482         if not cmsg:
0483             cmsgfile, cmsgfile_orig = get_commit_message_file_path(user)
0484         else:
0485             cmsg += " " + fmt_commit_user(user)
0486         added, apaths = aconf.vcs.add(catpaths_byconf[aconf], repadd=True)
0487         if not added:
0488             if onabortf:
0489                 onabortf()
0490             error(_("@info",
0491                     "VCS reports that some catalogs cannot be added."))
0492         cpaths = sorted(set(map(join_ncwd, catpaths_byconf[aconf] + apaths)))
0493         if not aconf.vcs.commit(cpaths, message=cmsg, msgfile=cmsgfile,
0494                                 incparents=False):
0495             if onabortf:
0496                 onabortf()
0497             if not cmsgfile:
0498                 error(_("@info",
0499                         "VCS reports that some catalogs cannot be committed."))
0500             else:
0501                 os.remove(cmsgfile)
0502                 error(_("@info",
0503                         "VCS reports that some catalogs cannot be committed "
0504                         "(commit message preserved in '%(file)s').",
0505                         file=cmsgfile_orig))
0506         if cmsgfile:
0507             os.remove(cmsgfile)
0508             os.remove(cmsgfile_orig)
0509 
0510 
0511 def fmt_commit_user (user):
0512 
0513     return "[>%s]" % user
0514 
0515 
0516 def get_commit_message_file_path (user):
0517 
0518     while True:
0519         tfmt = time.strftime("%Y-%m-%d-%H-%M-%S")
0520         prefix = "poascribe-commit-message"
0521         ext = "txt"
0522         fpath = "%s-%s.%s" % (prefix, tfmt, ext)
0523         fpath_asc = "%s-%s-asc.%s" % (prefix, tfmt, ext)
0524         if not os.path.isfile(fpath) and not os.path.isfile(fpath_asc):
0525             break
0526 
0527     edcmd = None
0528     if not edcmd:
0529         edcmd = os.getenv("ASC_EDITOR")
0530     if not edcmd:
0531         edcmd = pology_config.section("poascribe").string("editor")
0532     if not edcmd:
0533         edcmd = os.getenv("EDITOR")
0534     if not edcmd:
0535         edcmd = "/usr/bin/vi"
0536 
0537     cmd = "%s %s" % (edcmd, fpath)
0538     if os.system(cmd):
0539         error(_("@info",
0540                 "Еrror from editor command '%(cmd)s' for commit message.",
0541                 cmd=cmd))
0542     if not os.path.isfile(fpath):
0543         error(_("@info",
0544                 "Editor command '%(cmd)s' did not produce a file.",
0545                 cmd=cmd))
0546 
0547     cmsg = open(fpath, "r").read()
0548     if not cmsg.endswith("\n"):
0549         cmsg += "\n"
0550     fmt_user = unicode_to_str(fmt_commit_user(user))
0551     if cmsg.count("\n") == 1:
0552         cmsg = cmsg[:-1] + " " + fmt_user + "\n"
0553     else:
0554         cmsg += fmt_user + "\n"
0555     fh = open(fpath_asc, "w")
0556     fh.write(cmsg)
0557     fh.close()
0558 
0559     return fpath_asc, fpath
0560 
0561 
0562 def assert_mode_user (aconfs_catpaths, mode):
0563 
0564     for aconf, catpaths in aconfs_catpaths:
0565         if mode.user not in aconf.users:
0566             error(_("@info",
0567                     "User '%(user)s' not defined in '%(file)s'.",
0568                     user=mode.user, file=aconf.path))
0569 
0570 
0571 def assert_review_tags (aconfs_catpaths, tags):
0572 
0573     for aconf, catpaths in aconfs_catpaths:
0574         for tag in tags:
0575             if tag not in aconf.revtags:
0576                 error(_("@info",
0577                         "Review tag '%(tag)s' not defined in '%(file)s'.",
0578                         tag=tag, file=aconf.path))
0579 
0580 
0581 def assert_syntax (aconfs_catpaths, onabortf=None):
0582 
0583     checkf = msgfmt(options=["--check"])
0584     numerr = 0
0585     for aconf, catpaths in aconfs_catpaths:
0586         for catpath, acatpath in catpaths:
0587             numerr += checkf(catpath)
0588     if numerr:
0589         if onabortf:
0590             onabortf()
0591         error(_("@info",
0592                 "Invalid syntax in some files, see the reports above. "
0593                 "Ascription aborted."))
0594     return numerr
0595 
0596 
0597 def setup_progress (aconfs_catpaths, addfmt):
0598 
0599     acps = [y[0] for x in aconfs_catpaths for y in x[1]]
0600     return init_file_progress(acps, addfmt=addfmt)
0601 
0602 
0603 # Exclusive states of a message, as reported by Message.state().
0604 _st_tran = "T"
0605 _st_fuzzy = "F"
0606 _st_untran = "U"
0607 _st_otran = "OT"
0608 _st_ofuzzy = "OF"
0609 _st_ountran = "OU"
0610 _all_states = (
0611     _st_tran, _st_fuzzy, _st_untran,
0612     _st_otran, _st_ofuzzy, _st_ountran,
0613 )
0614 
0615 
0616 def status (options, aconfs_catpaths, mode):
0617 
0618     # Count ascribed and unascribed messages through catalogs.
0619     counts_a = dict([(x, {}) for x in _all_states])
0620     counts_na = dict([(x, {}) for x in _all_states])
0621 
0622     upprog = setup_progress(aconfs_catpaths,
0623                             t_("@info:progress",
0624                                "Examining state: %(file)s"))
0625     for aconf, catpaths in aconfs_catpaths:
0626         for catpath, acatpath in catpaths:
0627             upprog(catpath)
0628             # Open current and ascription catalog.
0629             cat = Catalog(catpath, monitored=False)
0630             acat = Catalog(acatpath, create=True, monitored=False)
0631             # Count ascribed and non-ascribed by original catalog.
0632             nselected = 0
0633             for msg in cat:
0634                 purge_msg(msg)
0635                 ahist = collect_ascription_history(
0636                     msg, acat, aconf,
0637                     hfilter=options.hfilter, addrem=options.addrem, nomrg=True)
0638                 if ahist[0].user is None and not has_tracked_parts(msg):
0639                     continue # pristine
0640                 if not mode.selector(msg, cat, ahist, aconf):
0641                     continue # not selected
0642                 counts = ahist[0].user is None and counts_na or counts_a
0643                 st = msg.state()
0644                 if catpath not in counts[st]:
0645                     counts[st][catpath] = 0
0646                 counts[st][catpath] += 1
0647                 nselected += 1
0648             # Count non-ascribed by ascription catalog.
0649             for amsg in acat:
0650                 if amsg not in cat:
0651                     ast = amsg.state()
0652                     st = None
0653                     if ast == _st_tran:
0654                         st = _st_otran
0655                     elif ast == _st_fuzzy:
0656                         st = _st_ofuzzy
0657                     elif ast == _st_untran:
0658                         st = _st_ountran
0659                     if st:
0660                         if catpath not in counts_na[st]:
0661                             counts_na[st][catpath] = 0
0662                         counts_na[st][catpath] += 1
0663             # Cancel counts if maximum selection fraction exceeded.
0664             if float(nselected) / len(cat) > options.max_fraction_select:
0665                 for counts in (counts_a, counts_na):
0666                     for st in _all_states:
0667                         if catpath in counts[st]:
0668                             counts[st].pop(catpath)
0669     upprog()
0670 
0671     # Some general data for tabulation of output.
0672     coln = [_("@title:column translated messages", "msg/t"),
0673             _("@title:column fuzzy messages", "msg/f"),
0674             _("@title:column untranslated messages", "msg/u"),
0675             _("@title:column obsolete translated messages", "msg/ot"),
0676             _("@title:column obsolete fuzzy messages", "msg/of"),
0677             _("@title:column obsolete untranslated messages", "msg/ou")]
0678     none="-"
0679 
0680     # NOTE: When reporting, do not show anything if there are
0681     # neither ascribed nor non-ascribed messages selected.
0682     # If there are some ascribed and none non-ascribed,
0683     # show only the row for ascribed.
0684     # However, if there are some non-ascribed but none ascribed,
0685     # still show the row for ascribed, to not accidentally confuse
0686     # non-ascribed for ascribed.
0687 
0688     # Report totals.
0689     totals_a, totals_na = {}, {}
0690     for totals, counts in ((totals_a, counts_a), (totals_na, counts_na)):
0691         for st, cnt_per_cat in list(counts.items()):
0692             totals[st] = sum(cnt_per_cat.values())
0693     # See previous NOTE.
0694     if sum(totals_a.values()) > 0 or sum(totals_na.values()) > 0:
0695         rown = [_("@title:row number of ascribed messages",
0696                   "ascribed")]
0697         data = [[totals_a[x] or None] for x in _all_states]
0698         if sum(totals_na.values()) > 0:
0699             rown.append(_("@title:row number of unascribed messages",
0700                           "unascribed"))
0701             for i in range(len(_all_states)):
0702                 data[i].append(totals_na[_all_states[i]] or None)
0703         report(tabulate(data=data, coln=coln, rown=rown,
0704                         none=none, colorize=True))
0705 
0706     # Report counts per catalog if requested.
0707     if options.show_by_file:
0708         catpaths = set()
0709         for counts in (counts_a, counts_na):
0710             catpaths.update(sum([list(x.keys()) for x in list(counts.values())], []))
0711         catpaths = sorted(catpaths)
0712         if catpaths:
0713             coln.insert(0, _("@title:column", "catalog"))
0714             coln.insert(1, _("@title:column state (asc/nasc)", "st"))
0715             data = [[] for x in _all_states]
0716             for catpath in catpaths:
0717                 cc_a = [counts_a[x].get(catpath, 0) for x in _all_states]
0718                 cc_na = [counts_na[x].get(catpath, 0) for x in _all_states]
0719                 # See previous NOTE.
0720                 if sum(cc_a) > 0 or sum(cc_na) > 0:
0721                     data[0].append(catpath)
0722                     data[1].append(
0723                         _("@item:intable number of ascribed messages",
0724                           "asc"))
0725                     for datac, cc in zip(data[2:], cc_a):
0726                         datac.append(cc or None)
0727                     if sum(cc_na) > 0:
0728                         data[0].append("^^^")
0729                         data[1].append(
0730                             _("@item:intable number of unascribed messages",
0731                               "nasc"))
0732                         for datac, cc in zip(data[2:], cc_na):
0733                             datac.append(cc or None)
0734             if any(data):
0735                 dfmt = ["%%-%ds" % max([len(x) for x in catpaths])]
0736                 report("-")
0737                 report(tabulate(data=data, coln=coln, dfmt=dfmt,
0738                                 none=none, colorize=True))
0739 
0740 
0741 # FIXME: Factor out into message module.
0742 _fields_current = (
0743     "msgctxt", "msgid", "msgid_plural",
0744 )
0745 _fields_previous = (
0746     "msgctxt_previous", "msgid_previous", "msgid_plural_previous",
0747 )
0748 
0749 def msg_to_previous (msg, copy=True):
0750 
0751     if msg.fuzzy and msg.msgid_previous is not None:
0752         pmsg = MessageUnsafe(msg) if copy else msg
0753         for fcurr, fprev in zip(_fields_current, _fields_previous):
0754             setattr(pmsg, fcurr, pmsg.get(fprev))
0755         pmsg.unfuzzy()
0756         return pmsg
0757 
0758 
0759 def restore_reviews (aconfs_catpaths, revspecs_by_catmsg):
0760 
0761     upprog = setup_progress(aconfs_catpaths,
0762                             t_("@info:progress",
0763                                "Restoring reviews: %(file)s"))
0764     nrestored = 0
0765     for aconf, catpaths in aconfs_catpaths:
0766         for catpath, acatpath in catpaths:
0767             upprog(catpath)
0768             revels_by_msg = revspecs_by_catmsg.get(catpath)
0769             if revels_by_msg:
0770                 cat = Catalog(catpath, monitored=True)
0771                 for msgref, revels in sorted(revels_by_msg.items()):
0772                     msg = cat[msgref - 1]
0773                     revtags, unrevd, revok = revels
0774                     restore_review_flags(msg, revtags, unrevd)
0775                     nrestored += 1
0776                 sync_and_rep(cat, shownmod=False)
0777                 if aconf.vcs.is_versioned(acatpath):
0778                     aconf.vcs.revert(acatpath)
0779                 # ...no else: because revert may cause the file
0780                 # not to be versioned any more.
0781                 if not aconf.vcs.is_versioned(acatpath):
0782                     os.remove(acatpath)
0783 
0784     if nrestored > 0:
0785         report(n_("@info:progress",
0786                   "===== Review elements restored to %(num)d message.",
0787                   "===== Review elements restored to %(num)d messages.",
0788                   num=nrestored))
0789 
0790 
0791 def restore_review_flags (msg, revtags, unrevd):
0792 
0793     for tag in revtags:
0794         flag = _revdflags[0]
0795         if tag:
0796             flag += _flagtagsep + tag
0797         msg.flag.add(flag)
0798     if unrevd:
0799         msg.flag.add(_urevdflags[0])
0800 
0801     return msg
0802 
0803 
0804 def commit (options, aconfs_catpaths, mode):
0805 
0806     assert_mode_user(aconfs_catpaths, mode)
0807 
0808     # Ascribe modifications and reviews.
0809     upprog = setup_progress(aconfs_catpaths,
0810                             t_("@info:progress",
0811                                "Ascribing: %(file)s"))
0812     revels = {}
0813     counts = dict([(x, [0, 0]) for x in _all_states])
0814     aconfs_catpaths_ascmod = []
0815     aconf_by_catpath = {}
0816     for aconf, catpaths in aconfs_catpaths:
0817         aconfs_catpaths_ascmod.append((aconf, []))
0818         for catpath, acatpath in catpaths:
0819             upprog(catpath)
0820             res = commit_cat(options, aconf, mode.user, catpath, acatpath,
0821                              mode.selector)
0822             ccounts, crevels, catmod = res
0823             for st, (nmod, nrev) in list(ccounts.items()):
0824                 counts[st][0] += nmod
0825                 counts[st][1] += nrev
0826             revels[catpath] = crevels
0827             if catmod:
0828                 aconfs_catpaths_ascmod[-1][1].append((catpath, acatpath))
0829             aconf_by_catpath[catpath] = aconf
0830     upprog()
0831 
0832     onabortf = lambda: restore_reviews(aconfs_catpaths_ascmod, revels)
0833 
0834     # Assert that all reviews were good.
0835     unknown_revtags = []
0836     for catpath, revels1 in sorted(revels.items()):
0837         aconf = aconf_by_catpath[catpath]
0838         for msgref, (revtags, unrevd, revok) in sorted(revels1.items()):
0839             if not revok:
0840                 onabortf()
0841                 error("Ascription aborted due to earlier warnings.")
0842 
0843     assert_syntax(aconfs_catpaths_ascmod, onabortf=onabortf)
0844     # ...must be done after committing, to have all review elements purged
0845 
0846     coln = [_("@title:column number of modified messages",
0847               "modified")]
0848     rown = []
0849     data = [[]]
0850     for st, stlabel in (
0851         (_st_tran,
0852          _("@title:row number of translated messages",
0853            "translated")),
0854         (_st_fuzzy,
0855          _("@title:row number of fuzzy messages",
0856            "fuzzy")),
0857         (_st_untran,
0858          _("@title:row number of untranslated messages",
0859            "untranslated")),
0860         (_st_otran,
0861          _("@title:row number of obsolete translated messages",
0862            "obsolete/t")),
0863         (_st_ofuzzy,
0864          _("@title:row number of obsolete fuzzy messages",
0865            "obsolete/f")),
0866         (_st_ountran,
0867          _("@title:row number of obsolete untranslated messages",
0868            "obsolete/u")),
0869     ):
0870         if counts[st][1] > 0 and len(coln) < 2:
0871             coln.append(_("@title:column number of reviewed messages",
0872                           "reviewed"))
0873             data.append([])
0874         if counts[st][0] > 0 or counts[st][1] > 0:
0875             rown.append(stlabel)
0876             data[0].append(counts[st][0] or None)
0877             if len(coln) >= 2:
0878                 data[1].append(counts[st][1] or None)
0879     if rown:
0880         report(_("@info:progress", "===== Ascription summary:"))
0881         report(tabulate(data, coln=coln, rown=rown, none="-",
0882                         colorize=True))
0883 
0884     if options.vcs_commit:
0885         vcs_commit_catalogs(aconfs_catpaths, mode.user,
0886                             message=options.message, onabortf=onabortf)
0887         # ...not configs_catpaths_ascmod, as non-ascription relevant
0888         # modifications may exist (e.g. new pristine catalog added).
0889 
0890 
0891 def diff (options, aconfs_catpaths, mode):
0892 
0893     upprog = setup_progress(aconfs_catpaths,
0894                             t_("@info:progress",
0895                                "Diffing for review: %(file)s"))
0896     ndiffed = 0
0897     for aconf, catpaths in aconfs_catpaths:
0898         for catpath, acatpath in catpaths:
0899             upprog(catpath)
0900             ndiffed += diff_cat(options, aconf, catpath, acatpath,
0901                                 mode.selector, mode.aselector)
0902     upprog()
0903     if ndiffed > 0:
0904         report(n_("@info:progress",
0905                   "===== %(num)d message diffed for review.",
0906                   "===== %(num)d messages diffed for review.",
0907                   num=ndiffed))
0908 
0909 
0910 def purge (options, aconfs_catpaths, mode):
0911 
0912     upprog = setup_progress(aconfs_catpaths,
0913                             t_("@info:progress",
0914                                "Purging review elements: %(file)s"))
0915     npurged = 0
0916     for aconf, catpaths in aconfs_catpaths:
0917         for catpath, acatpath in catpaths:
0918             upprog(catpath)
0919             npurged += purge_cat(options, aconf, catpath, acatpath,
0920                                  mode.selector)
0921     upprog()
0922 
0923     if npurged > 0:
0924         if not options.keep_flags:
0925             report(n_("@info:progress",
0926                       "===== Review elements purged from %(num)d message.",
0927                       "===== Review elements purged from %(num)d messages.",
0928                       num=npurged))
0929         else:
0930             report(n_("@info:progress",
0931                       "===== Review elements purged from %(num)d message "
0932                       "(flags kept).",
0933                       "===== Review elements purged from %(num)d messages "
0934                       "(flags kept).",
0935                       num=npurged))
0936 
0937     return npurged
0938 
0939 
0940 def history (options, aconfs_catpaths, mode):
0941 
0942     upprog = setup_progress(aconfs_catpaths,
0943                             t_("@info:progress",
0944                                "Computing histories: %(file)s"))
0945     nshown = 0
0946     for aconf, catpaths in aconfs_catpaths:
0947         for catpath, acatpath in catpaths:
0948             upprog(catpath)
0949             nshown += history_cat(options, aconf, catpath, acatpath,
0950                                   mode.selector)
0951     upprog()
0952     if nshown > 0:
0953         report(n_("@info:progress",
0954                   "===== Histories computed for %(num)d message.",
0955                   "===== Histories computed for %(num)d messages.",
0956                   num=nshown))
0957 
0958 
0959 def commit_cat (options, aconf, user, catpath, acatpath, stest):
0960 
0961     # Open current catalog and ascription catalog.
0962     # Monitored, for removal of review elements.
0963     cat = Catalog(catpath, monitored=True)
0964     acat = prep_write_asc_cat(acatpath, aconf)
0965 
0966     revtags_ovr = None
0967     if options.all_reviewed:
0968         revtags_ovr = options.tags
0969 
0970     # Collect unascribed messages, but ignoring pristine ones
0971     # (those which are both untranslated and without history).
0972     # Collect and purge any review elements.
0973     # Check if any modification cannot be due to merging
0974     # (if header update is requested).
0975     mod_msgs = []
0976     rev_msgs = []
0977     revels_by_msg = {}
0978     counts = dict([(x, [0, 0]) for x in _all_states])
0979     counts0 = counts.copy()
0980     any_nonmerges = False
0981     prev_msgs = []
0982     check_mid_msgs = []
0983     for msg in cat:
0984         mod, revtags, unrevd = purge_msg(msg)
0985         if mod:
0986             revels_by_msg[msg.refentry] = [revtags, unrevd, True]
0987         ahist = collect_ascription_history(msg, acat, aconf) # after purging
0988         # Do not ascribe anything if the message is new and untranslated.
0989         if (    ahist[0].user is None and len(ahist) == 1
0990             and not has_tracked_parts(msg)
0991         ):
0992             continue
0993         # Possibly ascribe review only if the message passes the selector.
0994         if stest(msg, cat, ahist, aconf) and (mod or revtags_ovr):
0995             if revtags_ovr:
0996                 revtags = revtags_ovr
0997                 unrevd = False
0998             if revtags and not unrevd: # unreviewed flag overrides
0999                 rev_msgs.append((msg, revtags))
1000                 counts[msg.state()][1] += 1
1001                 # Check and record if review tags are not valid.
1002                 unknown_revtags = revtags.difference(aconf.revtags)
1003                 if unknown_revtags:
1004                     revels_by_msg[msg.refentry][-1] = False
1005                     tagfmt = format_item_list(sorted(unknown_revtags))
1006                     warning_on_msg(_("@info",
1007                                      "Unknown review tags: %(taglist)s.",
1008                                      taglist=tagfmt), msg, cat)
1009         # Ascribe modification regardless of the selector.
1010         if ahist[0].user is None:
1011             mod_msgs.append(msg)
1012             counts[msg.state()][0] += 1
1013             if options.update_headers and not any_nonmerges:
1014                 if len(ahist) == 1 or not merge_modified(ahist[1].msg, msg):
1015                     any_nonmerges = True
1016             # Record that reconstruction of the post-merge message
1017             # should be tried if this message has no prior history
1018             # but it is not pristine (it may be that the translator
1019             # has merged the catalog and updated fuzzy messages in one step,
1020             # without committing the catalog right after merging).
1021             if len(ahist) == 1:
1022                 check_mid_msgs.append(msg)
1023         # Collect latest historical version of the message,
1024         # in case reconstruction of post-merge messages is needed.
1025         if ahist[0].user is not None or len(ahist) > 1:
1026             pmsg = ahist[1 if ahist[0].user is None else 0].msg
1027             prev_msgs.append(pmsg)
1028 
1029     # Collect non-obsolete ascribed messages that no longer have
1030     # original counterpart, to ascribe as obsolete.
1031     # If reconstruction of post-merge messages is needed,
1032     # also collect latest historical versions.
1033     cat.sync_map() # in case key fields were purged
1034     for amsg in acat:
1035         if amsg not in cat:
1036             ast = amsg.state()
1037             st = None
1038             if ast == _st_tran:
1039                 st = _st_otran
1040             elif ast == _st_fuzzy:
1041                 st = _st_ofuzzy
1042             elif ast == _st_untran:
1043                 st = _st_ountran
1044             if st or check_mid_msgs:
1045                 msg = collect_ascription_history_segment(amsg, acat, aconf)[0].msg
1046                 if check_mid_msgs:
1047                     prev_msgs.append(msg)
1048                 if st:
1049                     msg.obsolete = True
1050                     mod_msgs.append(msg)
1051                     counts[st][0] += 1
1052 
1053     # Shortcut if nothing to do, because sync_and_rep later are expensive.
1054     if not mod_msgs and not revels_by_msg:
1055         # No messages to commit.
1056         return counts0, revels_by_msg, False
1057 
1058     # Construct post-merge messages.
1059     mod_mid_msgs = []
1060     if check_mid_msgs and not acat.created():
1061         mid_cat = create_post_merge_cat(cat, prev_msgs)
1062         for msg in check_mid_msgs:
1063             mid_msg = mid_cat.get(msg)
1064             if (    mid_msg is not None
1065                 and mid_msg.fuzzy
1066                 and not ascription_equal(mid_msg, msg)
1067             ):
1068                 mod_mid_msgs.append(mid_msg)
1069 
1070     # Ascribe modifications.
1071     for mid_msg in mod_mid_msgs: # ascribe post-merge before actual
1072         ascribe_modification(mid_msg, user, _dt_start, acat, aconf)
1073     for msg in mod_msgs:
1074         ascribe_modification(msg, user, _dt_start, acat, aconf)
1075 
1076     # Ascribe reviews.
1077     for msg, revtags in rev_msgs:
1078         ascribe_review(msg, user, _dt_start, revtags, acat, aconf)
1079 
1080     # Update header if requested and translator's modifications detected.
1081     if options.update_headers and any_nonmerges:
1082         cat.update_header(project=cat.name,
1083                           title=aconf.title,
1084                           name=aconf.users[user].name,
1085                           email=aconf.users[user].email,
1086                           teamemail=aconf.teamemail,
1087                           langname=aconf.langteam,
1088                           langcode=aconf.langcode,
1089                           plforms=aconf.plforms)
1090 
1091     nmod = [len(mod_msgs)]
1092     if len(rev_msgs) > 0:
1093         nmod.append(len(rev_msgs))
1094     catmod = False
1095     if sync_and_rep(cat, nmod=nmod):
1096         catmod = True
1097     if asc_sync_and_rep(acat, shownmod=False, nmod=[0]):
1098         catmod = True
1099 
1100     return counts, revels_by_msg, catmod
1101 
1102 
1103 def diff_cat (options, aconf, catpath, acatpath, stest, aselect):
1104 
1105     cat = Catalog(catpath, monitored=True)
1106     acat = Catalog(acatpath, create=True, monitored=False)
1107 
1108     # Select messages for diffing.
1109     msgs_to_diff = []
1110     for msg in cat:
1111         purge_msg(msg)
1112         ahist = collect_ascription_history(
1113             msg, acat, aconf,
1114             hfilter=options.hfilter, addrem=options.addrem, nomrg=True)
1115         # Makes no sense to review pristine messages.
1116         if ahist[0].user is None and not has_tracked_parts(msg):
1117             continue
1118         sres = stest(msg, cat, ahist, aconf)
1119         if not sres:
1120             continue
1121         msgs_to_diff.append((msg, ahist, sres))
1122 
1123     # Cancel selection if maximum fraction exceeded.
1124     if float(len(msgs_to_diff)) / len(cat) > options.max_fraction_select:
1125         msgs_to_diff = []
1126 
1127     if not msgs_to_diff:
1128         return 0
1129 
1130     # Diff selected messages.
1131     diffed_msgs = []
1132     tagfmt = _flagtagsep.join(options.tags)
1133     for msg, ahist, sres in msgs_to_diff:
1134 
1135         # Try to select ascription to differentiate from.
1136         # (Note that ascription indices returned by selectors are 1-based.)
1137         i_asc = None
1138         if aselect:
1139             asres = aselect(msg, cat, ahist, aconf)
1140             i_asc = (asres - 1) if asres else None
1141         elif not isinstance(sres, bool):
1142             # If there is no ascription selector, but basic selector returned
1143             # an ascription index, use first earlier non-fuzzy for diffing.
1144             i_asc = sres - 1
1145             i_asc = first_non_fuzzy(ahist, i_asc + 1)
1146 
1147         # Differentiate and flag.
1148         amsg = i_asc is not None and ahist[i_asc].msg or None
1149         if amsg is not None:
1150             if editprob(amsg.msgid, msg.msgid) > options.min_adjsim_diff:
1151                 msg_ediff(amsg, msg, emsg=msg, pfilter=options.sfilter)
1152                 flag = _diffflag
1153             else:
1154                 # If to great difference, add special flag and do not diff.
1155                 flag = _diffflag_ign
1156         else:
1157             # If no previous ascription selected, add special flag.
1158             flag = _diffflag_tot
1159         if tagfmt:
1160             flag += _flagtagsep + tagfmt
1161         msg.flag.add(flag)
1162 
1163         # Add ascription chain comment.
1164         ascfmts = []
1165         i_from = (i_asc - 1) if i_asc is not None else len(ahist) - 1
1166         for i in range(i_from, -1, -1):
1167             a = ahist[i]
1168             shtype = {AscPoint.ATYPE_MOD: "m",
1169                       AscPoint.ATYPE_REV: "r"}[a.type]
1170             if a.tag:
1171                 ascfmt = "%s:%s(%s)" % (a.user, shtype, a.tag)
1172             else:
1173                 ascfmt = "%s:%s" % (a.user, shtype)
1174             ascfmts.append(ascfmt)
1175         achnfmt = "%s %s" % (_achncmnt, " ".join(ascfmts))
1176         msg.auto_comment.append(achnfmt)
1177 
1178         diffed_msgs.append(msg)
1179 
1180     sync_and_rep(cat)
1181 
1182     # Open in the PO editor if requested.
1183     if options.po_editor:
1184         for msg in diffed_msgs:
1185             options.po_editor(msg, cat,
1186                               report=_("@info note on selected message",
1187                                        "Selected for review."))
1188 
1189     return len(diffed_msgs)
1190 
1191 
1192 _subreflags = "|".join(_all_flags)
1193 _subrecmnts = "|".join(_all_cmnts)
1194 _any_to_purge_rx = re.compile(r"^\s*(#,.*\b(%s)|#\.\s*(%s))"
1195                               % (_subreflags, _subrecmnts),
1196                               re.M|re.U)
1197 
1198 # Quickly check if it may be that some messages in the PO file
1199 # have review elements (diffs, flags).
1200 def may_have_revels (catpath):
1201 
1202     return bool(_any_to_purge_rx.search(open(catpath).read()))
1203 
1204 
1205 def purge_cat (options, aconf, catpath, acatpath, stest):
1206 
1207     if not may_have_revels(catpath):
1208         return 0
1209 
1210     cat = Catalog(catpath, monitored=True)
1211     acat = Catalog(acatpath, create=True, monitored=False)
1212 
1213     # Select messages to purge.
1214     msgs_to_purge = []
1215     for msg in cat:
1216         cmsg = MessageUnsafe(msg)
1217         purge_msg(cmsg)
1218         ahist = collect_ascription_history(
1219             cmsg, acat, aconf,
1220             hfilter=options.hfilter, addrem=options.addrem, nomrg=True)
1221         if not stest(cmsg, cat, ahist, aconf):
1222             continue
1223         msgs_to_purge.append(msg)
1224 
1225     # Does observing options.max_fraction_select makes sense for purging?
1226     ## Cancel selection if maximum fraction exceeded.
1227     #if float(len(msgs_to_purge)) / len(cat) > options.max_fraction_select:
1228         #msgs_to_purge = []
1229 
1230     # Purge selected messages.
1231     npurged = 0
1232     for msg in msgs_to_purge:
1233         res = purge_msg(msg, keepflags=options.keep_flags)
1234         mod, revtags, unrevd = res
1235         if mod:
1236             npurged += 1
1237 
1238     sync_and_rep(cat)
1239 
1240     return npurged
1241 
1242 
1243 def history_cat (options, aconf, catpath, acatpath, stest):
1244 
1245     cat = Catalog(catpath, monitored=False)
1246     acat = Catalog(acatpath, create=True, monitored=False)
1247 
1248     # Select messages for which to compute histories.
1249     msgs_to_hist = []
1250     for msg in cat:
1251         purge_msg(msg)
1252         ahist = collect_ascription_history(
1253             msg, acat, aconf,
1254             hfilter=options.hfilter, addrem=options.addrem, nomrg=True)
1255         if not stest(msg, cat, ahist, aconf):
1256             continue
1257         msgs_to_hist.append((msg, ahist))
1258 
1259     # Cancel selection if maximum fraction exceeded.
1260     if float(len(msgs_to_hist)) / len(cat) > options.max_fraction_select:
1261         msgs_to_hist = []
1262 
1263     # Compute histories for selected messages.
1264     for msg, ahist in msgs_to_hist:
1265 
1266         unasc = ahist[0].user is None
1267         if unasc:
1268             ahist.pop(0)
1269 
1270         hlevels = len(ahist)
1271         if options.depth is not None:
1272             hlevels = int(options.depth)
1273             if ahist[0].user is None:
1274                 hlevels += 1
1275             if hlevels > len(ahist):
1276                 hlevels = len(ahist)
1277 
1278         hinfo = []
1279         if hlevels > 0:
1280             hinfo += [_("@info:progress",
1281                         "<green>>>> History follows:</green>")]
1282             hfmt = "%%%dd" % len(str(hlevels))
1283         for i in range(hlevels):
1284             a = ahist[i]
1285             if a.type == AscPoint.ATYPE_MOD:
1286                 anote = _("@item:intable",
1287                           "<bold>#%(pos)d</bold> "
1288                           "modified by %(user)s on %(date)s",
1289                           pos=a.pos, user=a.user, date=a.date)
1290             elif a.type == AscPoint.ATYPE_REV:
1291                 if not a.tag:
1292                     anote = _("@item:intable",
1293                               "<bold>#%(pos)d</bold> "
1294                               "reviewed by %(user)s on %(date)s",
1295                               pos=a.pos, user=a.user, date=a.date)
1296                 else:
1297                     anote = _("@item:intable",
1298                               "<bold>#%(pos)d</bold> "
1299                               "reviewed (%(tag)s) by %(user)s on %(date)s",
1300                                pos=a.pos, user=a.user, tag=a.tag, date=a.date)
1301             else:
1302                 warning_on_msg(
1303                     _("@info",
1304                       "Unknown ascription type '%(type)s' found in history.",
1305                       type=a.type), msg, cat)
1306                 continue
1307             hinfo += [anote]
1308             if not a.type == AscPoint.ATYPE_MOD:
1309                 # Nothing more to show if this ascription is not modification.
1310                 continue
1311             i_next = i + 1
1312             if i_next == len(ahist):
1313                 # Nothing more to show at end of history.
1314                 continue
1315             dmsg = MessageUnsafe(a.msg)
1316             nmsg = ahist[i_next].msg
1317             if dmsg != nmsg:
1318                 msg_ediff(nmsg, dmsg, emsg=dmsg,
1319                           pfilter=options.sfilter, colorize=True)
1320                 dmsgfmt = dmsg.to_string(force=True,
1321                                          wrapf=cat.wrapf()).rstrip("\n")
1322                 hindent = " " * (len(hfmt % 0) + 2)
1323                 hinfo += [hindent + x for x in dmsgfmt.split("\n")]
1324         hinfo = cjoin(hinfo, "\n")
1325 
1326         if unasc or msg.fuzzy:
1327             pmsg = None
1328             i_nfasc = first_non_fuzzy(ahist)
1329             if i_nfasc is not None:
1330                 pmsg = ahist[i_nfasc].msg
1331             elif msg.fuzzy and msg.msgid_previous is not None:
1332                 pmsg = msg_to_previous(msg)
1333             if pmsg is not None:
1334                 for fprev in _fields_previous:
1335                     setattr(msg, fprev, None)
1336                 msg_ediff(pmsg, msg, emsg=msg,
1337                           pfilter=options.sfilter, colorize=True)
1338         report_msg_content(msg, cat,
1339                            note=(hinfo or None), delim=("-" * 20))
1340 
1341     return len(msgs_to_hist)
1342 
1343 
1344 _revflags_rx = re.compile(r"^(%s)(?: */(.*))?" % "|".join(_all_flags), re.I)
1345 
1346 def purge_msg (msg, keepflags=False):
1347 
1348     modified = False
1349 
1350     # Remove review flags.
1351     diffed = False
1352     revtags = set()
1353     unrevd = False
1354     for flag in list(msg.flag): # modified inside
1355         m = _revflags_rx.search(flag)
1356         if m:
1357             sflag = m.group(1)
1358             tagstr = m.group(2) or ""
1359             tags = [x.strip() for x in tagstr.split(_flagtagsep)]
1360             if sflag not in _urevdflags:
1361                 revtags.update(tags)
1362                 if sflag == _diffflag:
1363                     # ...must not check ...in _diffflags because with
1364                     # those other flags there is actually no diff.
1365                     diffed = True
1366             else:
1367                 unrevd = True
1368             msg.flag.remove(flag)
1369             modified = True
1370 
1371     # Remove review comments.
1372     i = 0
1373     while i < len(msg.auto_comment):
1374         cmnt = msg.auto_comment[i].strip()
1375         if cmnt.startswith(_all_cmnts):
1376             msg.auto_comment.pop(i)
1377             modified = True
1378         else:
1379             i += 1
1380 
1381     # Remove any leftover previous fields.
1382     if msg.translated:
1383         for fprev in _fields_previous:
1384             if msg.get(fprev) is not None:
1385                 setattr(msg, fprev, None)
1386                 modified = True
1387 
1388     if diffed:
1389         msg_ediff_to_new(msg, rmsg=msg)
1390     if keepflags:
1391         restore_review_flags(msg, revtags, unrevd)
1392 
1393     return modified, revtags, unrevd
1394 
1395 
1396 def prep_write_asc_cat (acatpath, aconf):
1397 
1398     if not os.path.isfile(acatpath):
1399         return init_asc_cat(acatpath, aconf)
1400     else:
1401         return Catalog(acatpath, monitored=True, wrapping=_ascwrapping)
1402 
1403 
1404 def init_asc_cat (acatpath, aconf):
1405 
1406     acat = Catalog(acatpath, create=True, monitored=True, wrapping=_ascwrapping)
1407     ahdr = acat.header
1408 
1409     ahdr.title = Monlist(["Ascription shadow for %s.po" % acat.name])
1410 
1411     translator = "Ascriber"
1412 
1413     if aconf.teamemail:
1414         author = "%s <%s>" % (translator, aconf.teamemail)
1415     else:
1416         author = "%s" % translator
1417     ahdr.author = Monlist([author])
1418 
1419     ahdr.copyright = "Copyright same as for the original catalog."
1420     ahdr.license = "License same as for the original catalog."
1421     ahdr.comment = Monlist(["===== DO NOT EDIT MANUALLY ====="])
1422 
1423     ahdr.set_field("Project-Id-Version", str(acat.name))
1424     ahdr.set_field("Report-Msgid-Bugs-To", str(aconf.teamemail or ""))
1425     ahdr.set_field("PO-Revision-Date", format_datetime(_dt_start))
1426     ahdr.set_field("Content-Type", "text/plain; charset=UTF-8")
1427     ahdr.set_field("Content-Transfer-Encoding", "8bit")
1428 
1429     if aconf.teamemail:
1430         ltr = "%s <%s>" % (translator, aconf.teamemail)
1431     else:
1432         ltr = translator
1433     ahdr.set_field("Last-Translator", str(ltr))
1434 
1435     if aconf.langteam:
1436         if aconf.teamemail:
1437             tline = "%s <%s>" % (aconf.langteam, aconf.teamemail)
1438         else:
1439             tline = aconf.langteam
1440         ahdr.set_field("Language-Team", str(tline))
1441     else:
1442         ahdr.remove_field("Language-Team")
1443 
1444     if aconf.langcode:
1445         ahdr.set_field("Language", str(aconf.langcode))
1446     else:
1447         ahdr.remove_field("Language")
1448 
1449     if aconf.plforms:
1450         ahdr.set_field("Plural-Forms", str(aconf.plforms))
1451     else:
1452         ahdr.remove_field("Plural-Forms")
1453 
1454     return acat
1455 
1456 
1457 def update_asc_hdr (acat):
1458 
1459     acat.header.set_field("PO-Revision-Date", format_datetime(_dt_start))
1460 
1461 
1462 def create_post_merge_cat (cat, prev_msgs):
1463 
1464     # Prepare previous catalog based on ascription catalog.
1465     prev_cat = Catalog("", create=True, monitored=False)
1466     prev_cat.header = Header(cat.header)
1467     for prev_msg in prev_msgs:
1468         prev_cat.add_last(prev_msg)
1469     tmpf1 = NamedTemporaryFile(prefix="pology-merged-", suffix=".po")
1470     prev_cat.filename = tmpf1.name
1471     prev_cat.sync()
1472 
1473     # Prepare template based on current catalog.
1474     tmpl_cat = Catalog("", create=True, monitored=False)
1475     tmpl_cat.header = Header(cat.header)
1476     for msg in cat:
1477         if not msg.obsolete:
1478             tmpl_msg = MessageUnsafe(msg)
1479             tmpl_msg.clear()
1480             tmpl_cat.add_last(tmpl_msg)
1481     tmpf2 = NamedTemporaryFile(prefix="pology-template-", suffix=".pot")
1482     tmpl_cat.filename = tmpf2.name
1483     tmpl_cat.sync()
1484 
1485     # Merge previous catalog using current catalog as template.
1486     mid_cat = merge_pofile(prev_cat.filename, tmpl_cat.filename,
1487                            getcat=True, monitored=False, quiet=True)
1488 
1489     return mid_cat
1490 
1491 
1492 _modified_cats = []
1493 
1494 def sync_and_rep (cat, shownmod=True, nmod=None):
1495 
1496     if shownmod and nmod is None:
1497         nmod = [0]
1498         for msg in cat:
1499             if msg.modcount:
1500                 nmod[0] += 1
1501 
1502     modified = cat.sync()
1503     if nmod and sum(nmod) > 0: # DO NOT check instead modified == True
1504         if shownmod:
1505             nmodfmt = "/".join("%d" % x for x in nmod)
1506             report("%s  (%s)" % (cat.filename, nmodfmt))
1507         else:
1508             report("%s" % cat.filename)
1509         _modified_cats.append(cat.filename)
1510 
1511     return modified
1512 
1513 
1514 def asc_sync_and_rep (acat, shownmod=True, nmod=None):
1515 
1516     if acat.modcount:
1517         update_asc_hdr(acat)
1518         mkdirpath(os.path.dirname(acat.filename))
1519 
1520     return sync_and_rep(acat, shownmod=shownmod, nmod=nmod)
1521 
1522 
1523 # -----------------------------------------------------------------------------
1524 
1525 if __name__ == "__main__":
1526     exit_on_exception(main)