Warning, /sdk/pology/bin/poascribe is written in an unsupported language. File is not indexed.
0001 #!/usr/bin/env python3
0002 # -*- coding: UTF-8 -*-
0003
0004 try:
0005 import fallback_import_paths
0006 except:
0007 pass
0008
0009 import datetime
0010 import locale
0011 import os
0012 import re
0013 import sys
0014 from tempfile import NamedTemporaryFile
0015 import time
0016
0017 from pology import PologyError, version, _, n_, t_
0018 from pology.ascript import collect_ascription_associations
0019 from pology.ascript import collect_ascription_history
0020 from pology.ascript import collect_ascription_history_segment
0021 from pology.ascript import ascription_equal, merge_modified
0022 from pology.ascript import ascribe_modification, ascribe_review
0023 from pology.ascript import first_non_fuzzy, has_tracked_parts
0024 from pology.ascript import make_ascription_selector
0025 from pology.ascript import AscPoint
0026 from pology.catalog import Catalog
0027 from pology.header import Header, TZInfo, format_datetime
0028 from pology.message import Message, MessageUnsafe
0029 from pology.gtxtools import msgfmt
0030 from pology.colors import ColorOptionParser, cjoin
0031 import pology.config as pology_config
0032 from pology.diff import msg_ediff, msg_ediff_to_new
0033 from pology.diff import editprob
0034 from pology.fsops import str_to_unicode, unicode_to_str
0035 from pology.fsops import collect_paths_cmdline, collect_catalogs
0036 from pology.fsops import mkdirpath, join_ncwd
0037 from pology.fsops import exit_on_exception
0038 from pology.getfunc import get_hook_ireq
0039 from pology.merge import merge_pofile
0040 from pology.monitored import Monlist
0041 from pology.msgreport import warning_on_msg, report_msg_content
0042 from pology.msgreport import report_msg_to_lokalize
0043 from pology.report import report, error, format_item_list
0044 from pology.report import init_file_progress
0045 from pology.stdcmdopt import add_cmdopt_incexc, add_cmdopt_filesfrom
0046 from pology.tabulate import tabulate
0047
0048
0049 # Wrapping in ascription catalogs.
0050 _ascwrapping = ["fine"]
0051
0052 # Flag used to mark diffed messages.
0053 # NOTE: All diff flags should start with 'ediff', as some other scripts
0054 # only need to check if any of them is present.
0055 _diffflag = "ediff"
0056 _diffflag_tot = "ediff-total"
0057 _diffflag_ign = "ediff-ignored"
0058
0059 # Flags used to explicitly mark messages as reviewed or unreviewed.
0060 _revdflags = ("reviewed", "revd", "rev") # synonyms
0061 _urevdflags = ("unreviewed", "nrevd", "nrev") # synonyms
0062
0063 # Comment used to show ascription chain in messages marked for review.
0064 _achncmnt = "~ascto:"
0065
0066 # String used to separate tags to review flags.
0067 _flagtagsep = "/"
0068
0069 _diffflags = (_diffflag, _diffflag_tot, _diffflag_ign)
0070 _all_flags = _diffflags + _revdflags + _urevdflags
0071 _all_flags = sorted(_all_flags, key=lambda x: (-len(x), x))
0072 # ...this order is necessary for proper |-linking in regexes.
0073 _all_cmnts = (_achncmnt,)
0074
0075 # Datetime at the moment the script is started.
0076 _dt_start = datetime.datetime(*(time.localtime()[:6] + (0, TZInfo())))
0077
0078
0079 def main ():
0080
0081 locale.setlocale(locale.LC_ALL, "")
0082
0083 mode_spec = (
0084 ("status", ("st",)),
0085 ("commit", ("co", "ci", "mo")),
0086 ("diff", ("di",)),
0087 ("purge", ("pu",)),
0088 ("history", ("hi",)),
0089 )
0090 mode_allnames = set()
0091 mode_tolong = {}
0092 for name, syns in mode_spec:
0093 mode_allnames.add(name)
0094 mode_allnames.update(syns)
0095 mode_tolong[name] = name
0096 mode_tolong.update((s, name) for s in syns)
0097
0098 known_editors = {
0099 "lokalize": report_msg_to_lokalize,
0100 }
0101
0102 # Setup options and parse the command line.
0103 usage = _("@info command usage",
0104 "%(cmd)s MODE [OPTIONS] [PATHS...]",
0105 cmd="%prog")
0106 desc = _("@info command description",
0107 "Keep track of who, when, and how, has translated, modified, "
0108 "or reviewed messages in a collection of PO files.")
0109 ver = _("@info command version",
0110 "%(cmd)s (Pology) %(version)s\n"
0111 "Copyright © 2008, 2009, 2010 "
0112 "Chusslove Illich (Часлав Илић) <%(email)s>",
0113 cmd="%prog", version=version(), email="caslav.ilic@gmx.net")
0114
0115 opars = ColorOptionParser(usage=usage, description=desc, version=ver)
0116 opars.add_option(
0117 "-a", "--select-ascription",
0118 metavar=_("@info command line value placeholder", "SELECTOR[:ARGS]"),
0119 action="append", dest="aselectors", default=None,
0120 help=_("@info command line option description",
0121 "Select a message from ascription history by this selector. "
0122 "Can be repeated, in which case the message is selected "
0123 "if all selectors match it."))
0124 opars.add_option(
0125 "-A", "--min-adjsim-diff",
0126 metavar=_("@info command line value placeholder", "RATIO"),
0127 action="store", dest="min_adjsim_diff", default=None,
0128 help=_("@info command line option description",
0129 "Minimum adjusted similarity between two versions of a message "
0130 "needed to show the embedded difference. "
0131 "Range 0.0-1.0, where 0 means always to show the difference, "
0132 "and 1 never to show it; a convenient range is 0.6-0.8. "
0133 "When the difference is not shown, the '%(flag)s' flag is "
0134 "added to the message.",
0135 flag=_diffflag_ign))
0136 opars.add_option(
0137 "-b", "--show-by-file",
0138 action="store_true", dest="show_by_file", default=False,
0139 help=_("@info command line option description",
0140 "Next to global summary, also present results by file."))
0141 opars.add_option(
0142 "-C", "--no-vcs-commit",
0143 action="store_false", dest="vcs_commit", default=None,
0144 help=_("@info command line option description",
0145 "Do not commit catalogs to version control "
0146 "(when version control is used)."))
0147 opars.add_option(
0148 "-d", "--depth",
0149 metavar=_("@info command line value placeholder", "LEVEL"),
0150 action="store", dest="depth", default=None,
0151 help=_("@info command line option description",
0152 "Consider ascription history up to this level into the past."))
0153 opars.add_option(
0154 "-D", "--diff-reduce-history",
0155 metavar=_("@info command line value placeholder", "SPEC"),
0156 action="store", dest="diff_reduce_history", default=None,
0157 help=_("@info command line option description",
0158 "Reduce each message in history to a part of the difference "
0159 "from the first earlier modification: to added, removed, or "
0160 "equal segments. "
0161 "The value begins with one of the characters 'a', 'r', or 'e', "
0162 "followed by substring that will be used to separate "
0163 "selected difference segments in resulting messages "
0164 "(if this substring is empty, space is used)."))
0165 opars.add_option(
0166 "-F", "--filter",
0167 metavar=_("@info command line value placeholder", "NAME"),
0168 action="append", dest="filters", default=None,
0169 help=_("@info command line option description",
0170 "Pass relevant message text fields through a filter before "
0171 "matching or comparing them (relevant in some modes). "
0172 "Can be repeated to add several filters."))
0173 opars.add_option(
0174 "-G", "--show-filtered",
0175 action="store_true", dest="show_filtered", default=False,
0176 help=_("@info command line option description",
0177 "When operating under a filter, also show filtered versions "
0178 "of whatever is shown in original (e.g. in diffs)."))
0179 opars.add_option(
0180 "-k", "--keep-flags",
0181 action="store_true", dest="keep_flags", default=False,
0182 help=_("@info command line option description",
0183 "Do not remove review-significant flags from messages "
0184 "(possibly convert them as appropriate)."))
0185 opars.add_option(
0186 "-m", "--message",
0187 metavar=_("@info command line value placeholder", "TEXT"),
0188 action="store", dest="message", default=None,
0189 help=_("@info command line option description",
0190 "Version control commit message for original catalogs, "
0191 "when %(opt)s is in effect.",
0192 opt="-c"))
0193 opars.add_option(
0194 "-o", "--open-in-editor",
0195 metavar=("|".join(sorted(known_editors))),
0196 action="store", dest="po_editor", default=None,
0197 help=_("@info command line option description",
0198 "Open selected messages in one of the supported PO editors."))
0199 opars.add_option(
0200 "-L", "--max-fraction-select",
0201 metavar=_("@info command line value placeholder", "FRACTION"),
0202 action="store", dest="max_fraction_select", default=None,
0203 help=_("@info command line option description",
0204 "Select messages in a catalog only if the total number "
0205 "of selected messages in that catalog would be at most "
0206 "the given fraction (0.0-1.0) of total number of messages."))
0207 opars.add_option(
0208 "-s", "--selector",
0209 metavar=_("@info command line value placeholder", "SELECTOR[:ARGS]"),
0210 action="append", dest="selectors", default=None,
0211 help=_("@info command line option description",
0212 "Consider only messages matched by this selector. "
0213 "Can be repeated, in which case the message is selected "
0214 "if all selectors match it."))
0215 opars.add_option(
0216 "-t", "--tag",
0217 metavar=_("@info command line value placeholder", "TAG"),
0218 action="store", dest="tags", default=None,
0219 help=_("@info command line option description",
0220 "Tag to add or consider in ascription records. "
0221 "Several tags may be given separated by commas."))
0222 opars.add_option(
0223 "-u", "--user",
0224 metavar=_("@info command line value placeholder", "USER"),
0225 action="store", dest="user", default=None,
0226 help=_("@info command line option description",
0227 "User in whose name the operation is performed."))
0228 opars.add_option(
0229 "-U", "--update-headers",
0230 action="store_true", dest="update_headers", default=None,
0231 help=_("@info command line option description",
0232 "Update headers in catalogs which contain modifications "
0233 "before committing them, with user's translator information."))
0234 opars.add_option(
0235 "-v", "--verbose",
0236 action="store_true", dest="verbose", default=False,
0237 help=_("@info command line option description",
0238 "Output more detailed progress info."))
0239 opars.add_option(
0240 "-w", "--write-modified",
0241 metavar=_("@info command line value placeholder", "FILE"),
0242 action="store", dest="write_modified", default=None,
0243 help=_("@info command line option description",
0244 "Write paths of all original catalogs modified by "
0245 "ascription operations into the given file."))
0246 opars.add_option(
0247 "-x", "--externals",
0248 metavar=_("@info command line value placeholder", "PYFILE"),
0249 action="append", dest="externals", default=[],
0250 help=_("@info command line option description",
0251 "Collect optional functionality from an external Python file "
0252 "(selectors, etc)."))
0253 opars.add_option(
0254 "--all-reviewed",
0255 action="store_true", dest="all_reviewed", default=False,
0256 help=_("@info command line option description",
0257 "Ascribe all messages as reviewed on commit, "
0258 "overriding any existing review elements. "
0259 "Tags given by %(opt)s apply. "
0260 "This should not be done in day-to-day practice; "
0261 "the primary use is initial review ascription.",
0262 opt="--tag"))
0263 add_cmdopt_filesfrom(opars)
0264 add_cmdopt_incexc(opars)
0265
0266 (options, free_args) = opars.parse_args(str_to_unicode(sys.argv[1:]))
0267
0268 # Parse operation mode and its arguments.
0269 if len(free_args) < 1:
0270 error(_("@info", "Operation mode not given."))
0271 rawmodename = free_args.pop(0)
0272 modename = mode_tolong.get(rawmodename)
0273 if modename is None:
0274 flatmodes = ["/".join((x[0],) + x[1]) for x in mode_spec]
0275 error(_("@info",
0276 "Unknown operation mode '%(mode)s' "
0277 "(known modes: %(modelist)s).",
0278 mode=rawmodename,
0279 modelist=format_item_list(flatmodes)))
0280
0281 # For options not issued, read values from user configuration.
0282 # Configuration values can also be issued by mode using
0283 # C{afield/amode = value} syntax, which takes precedence over
0284 # general fields (e.g. C{filters/review} vs. C{filters}).
0285 cfgsec = pology_config.section("poascribe")
0286 for optname, getvalf, defval in (
0287 ("aselectors", cfgsec.strdlist, []),
0288 ("vcs-commit", cfgsec.boolean, True),
0289 ("po-editor", cfgsec.string, None),
0290 ("filters", cfgsec.strslist, []),
0291 ("min-adjsim-diff", cfgsec.real, 0.0),
0292 ("selectors", cfgsec.strdlist, []),
0293 ("tags", cfgsec.string, ""),
0294 ("user", cfgsec.string, None),
0295 ("update-headers", cfgsec.boolean, False),
0296 ("diff-reduce-history", cfgsec.string, None),
0297 ("max-fraction-select", cfgsec.real, 1.01),
0298 ):
0299 uoptname = optname.replace("-", "_")
0300 if getattr(options, uoptname) is None:
0301 for fldname in ("%s/%s" % (optname, modename), optname):
0302 fldval = getvalf(fldname, None)
0303 if fldval is not None:
0304 break
0305 if fldval is None:
0306 fldval = defval
0307 setattr(options, uoptname, fldval)
0308
0309 # Convert options to non-string types.
0310 def valconv_editor (edkey):
0311 msgrepf = known_editors.get(edkey)
0312 if msgrepf is None:
0313 error(_("@info",
0314 "PO editor '%(ed)s' is not among "
0315 "the supported editors: %(edlist)s.",
0316 ed=edkey, edlist=format_item_list(sorted(known_editors))))
0317 return msgrepf
0318 def valconv_tags (cstr):
0319 return set(x.strip() for x in cstr.split(","))
0320 for optname, valconv in (
0321 ("max-fraction-select", float),
0322 ("min-adjsim-diff", float),
0323 ("po-editor", valconv_editor),
0324 ("tags", valconv_tags),
0325 ):
0326 uoptname = optname.replace("-", "_")
0327 valraw = getattr(options, uoptname, None)
0328 if valraw is not None:
0329 try:
0330 value = valconv(valraw)
0331 except TypeError:
0332 error(_("@info",
0333 "Value '%(val)s' to option '%(opt)s' is of wrong type.",
0334 val=valraw, opt=("--" + optname)))
0335 setattr(options, uoptname, value)
0336
0337 # Collect any external functionality.
0338 for xmod_path in options.externals:
0339 collect_externals(xmod_path)
0340
0341 # Create history filter if requested, store it in options.
0342 options.hfilter = None
0343 options.sfilter = None
0344 if options.filters:
0345 hfilters = []
0346 for hspec in options.filters:
0347 hfilters.append(get_hook_ireq(hspec, abort=True))
0348 def hfilter_composition (text):
0349 for hfilter in hfilters:
0350 text = hfilter(text)
0351 return text
0352 options.hfilter = hfilter_composition
0353 if options.show_filtered:
0354 options.sfilter = options.hfilter
0355
0356 # Create specification for reducing historical messages to diffs.
0357 options.addrem = None
0358 if options.diff_reduce_history:
0359 options.addrem = options.diff_reduce_history
0360 if options.addrem[:1] not in ("a", "e", "r"):
0361 error(_("@info",
0362 "Value '%(val)s' to option '%(opt)s' must start "
0363 "with '%(char1)s', '%(char2)s', or '%(char3)s'.",
0364 val=options.addrem, opt="--diff-reduce-history",
0365 char1="a", char2="e", char3="r"))
0366
0367 # Create selectors if any explicitly given.
0368 selector = None
0369 if options.selectors:
0370 selector = make_ascription_selector(options.selectors)
0371 aselector = None
0372 if options.aselectors:
0373 aselector = make_ascription_selector(options.aselectors, hist=True)
0374
0375 # Assemble operation mode.
0376 needuser = False
0377 canselect = False
0378 canaselect = False
0379 class _Mode: pass
0380 mode = _Mode()
0381 mode.name = modename
0382 if 0: pass
0383 elif mode.name == "status":
0384 mode.execute = status
0385 mode.selector = selector or make_ascription_selector(["any"])
0386 canselect = True
0387 elif mode.name == "commit":
0388 mode.execute = commit
0389 mode.selector = selector or make_ascription_selector(["any"])
0390 needuser = True
0391 canselect = True
0392 elif mode.name == "diff":
0393 mode.execute = diff
0394 mode.selector = selector or make_ascription_selector(["modar"])
0395 mode.aselector = aselector
0396 canselect = True
0397 canaselect = True
0398 elif mode.name == "purge":
0399 mode.execute = purge
0400 mode.selector = selector or make_ascription_selector(["any"])
0401 canselect = True
0402 elif mode.name == "history":
0403 mode.execute = history
0404 mode.selector = selector or make_ascription_selector(["any"])
0405 canselect = True
0406 else:
0407 error(_("@info",
0408 "Unhandled operation mode '%(mode)s'.",
0409 mode=mode.name))
0410
0411 mode.user = None
0412 if needuser:
0413 if not options.user:
0414 error(_("@info",
0415 "Operation mode '%(mode)s' requires a user "
0416 "to be specified.",
0417 mode=mode.name))
0418 mode.user = options.user
0419 if not canselect and selector:
0420 error(_("@info",
0421 "Operation mode '%(mode)s' does not accept selectors.",
0422 mode=mode.name))
0423 if not canaselect and aselector:
0424 error(_("@info",
0425 "Operation mode '%(mode)s' does not accept history selectors.",
0426 mode=mode.name))
0427
0428 # Collect list of catalogs supplied through command line.
0429 # If none supplied, assume current working directory.
0430 catpaths = collect_paths_cmdline(rawpaths=free_args,
0431 incnames=options.include_names,
0432 incpaths=options.include_paths,
0433 excnames=options.exclude_names,
0434 excpaths=options.exclude_paths,
0435 filesfrom=options.files_from,
0436 elsecwd=True,
0437 respathf=collect_catalogs,
0438 abort=True)
0439
0440 # Split catalogs into lists by ascription config,
0441 # and link them to their ascription catalogs.
0442 aconfs_catpaths = collect_ascription_associations(catpaths)
0443 assert_review_tags(aconfs_catpaths, options.tags)
0444
0445 # Execute operation.
0446 mode.execute(options, aconfs_catpaths, mode)
0447
0448 # Write out list of modified original catalogs if requested.
0449 if options.write_modified and _modified_cats:
0450 lfpath = options.write_modified
0451 f = open(lfpath, "w")
0452 f.write(("\n".join(sorted(_modified_cats)) + "\n").encode("utf-8"))
0453 f.close()
0454 report(_("@info",
0455 "Paths of modified catalogs written to '%(file)s'.",
0456 file=lfpath))
0457
0458
0459 def vcs_commit_catalogs (aconfs_catpaths, user, message=None, onabortf=None):
0460
0461 report(_("@info:progress VCS is acronym for \"version control system\"",
0462 ">>>>> VCS is committing catalogs:"))
0463
0464 # Attach paths to each distinct config, to commit them all at once.
0465 aconfs = []
0466 catpaths_byconf = {}
0467 for aconf, catpaths in aconfs_catpaths:
0468 if aconf not in catpaths_byconf:
0469 catpaths_byconf[aconf] = []
0470 aconfs.append(aconf)
0471 for catpath, acatpath in catpaths:
0472 catpaths_byconf[aconf].append(catpath)
0473 if os.path.isfile(acatpath):
0474 catpaths_byconf[aconf].append(acatpath)
0475
0476 # Commit by config.
0477 for aconf in aconfs:
0478 cmsg = message
0479 cmsgfile = None
0480 if not cmsg:
0481 cmsg = aconf.commitmsg
0482 if not cmsg:
0483 cmsgfile, cmsgfile_orig = get_commit_message_file_path(user)
0484 else:
0485 cmsg += " " + fmt_commit_user(user)
0486 added, apaths = aconf.vcs.add(catpaths_byconf[aconf], repadd=True)
0487 if not added:
0488 if onabortf:
0489 onabortf()
0490 error(_("@info",
0491 "VCS reports that some catalogs cannot be added."))
0492 cpaths = sorted(set(map(join_ncwd, catpaths_byconf[aconf] + apaths)))
0493 if not aconf.vcs.commit(cpaths, message=cmsg, msgfile=cmsgfile,
0494 incparents=False):
0495 if onabortf:
0496 onabortf()
0497 if not cmsgfile:
0498 error(_("@info",
0499 "VCS reports that some catalogs cannot be committed."))
0500 else:
0501 os.remove(cmsgfile)
0502 error(_("@info",
0503 "VCS reports that some catalogs cannot be committed "
0504 "(commit message preserved in '%(file)s').",
0505 file=cmsgfile_orig))
0506 if cmsgfile:
0507 os.remove(cmsgfile)
0508 os.remove(cmsgfile_orig)
0509
0510
0511 def fmt_commit_user (user):
0512
0513 return "[>%s]" % user
0514
0515
0516 def get_commit_message_file_path (user):
0517
0518 while True:
0519 tfmt = time.strftime("%Y-%m-%d-%H-%M-%S")
0520 prefix = "poascribe-commit-message"
0521 ext = "txt"
0522 fpath = "%s-%s.%s" % (prefix, tfmt, ext)
0523 fpath_asc = "%s-%s-asc.%s" % (prefix, tfmt, ext)
0524 if not os.path.isfile(fpath) and not os.path.isfile(fpath_asc):
0525 break
0526
0527 edcmd = None
0528 if not edcmd:
0529 edcmd = os.getenv("ASC_EDITOR")
0530 if not edcmd:
0531 edcmd = pology_config.section("poascribe").string("editor")
0532 if not edcmd:
0533 edcmd = os.getenv("EDITOR")
0534 if not edcmd:
0535 edcmd = "/usr/bin/vi"
0536
0537 cmd = "%s %s" % (edcmd, fpath)
0538 if os.system(cmd):
0539 error(_("@info",
0540 "Еrror from editor command '%(cmd)s' for commit message.",
0541 cmd=cmd))
0542 if not os.path.isfile(fpath):
0543 error(_("@info",
0544 "Editor command '%(cmd)s' did not produce a file.",
0545 cmd=cmd))
0546
0547 cmsg = open(fpath, "r").read()
0548 if not cmsg.endswith("\n"):
0549 cmsg += "\n"
0550 fmt_user = unicode_to_str(fmt_commit_user(user))
0551 if cmsg.count("\n") == 1:
0552 cmsg = cmsg[:-1] + " " + fmt_user + "\n"
0553 else:
0554 cmsg += fmt_user + "\n"
0555 fh = open(fpath_asc, "w")
0556 fh.write(cmsg)
0557 fh.close()
0558
0559 return fpath_asc, fpath
0560
0561
0562 def assert_mode_user (aconfs_catpaths, mode):
0563
0564 for aconf, catpaths in aconfs_catpaths:
0565 if mode.user not in aconf.users:
0566 error(_("@info",
0567 "User '%(user)s' not defined in '%(file)s'.",
0568 user=mode.user, file=aconf.path))
0569
0570
0571 def assert_review_tags (aconfs_catpaths, tags):
0572
0573 for aconf, catpaths in aconfs_catpaths:
0574 for tag in tags:
0575 if tag not in aconf.revtags:
0576 error(_("@info",
0577 "Review tag '%(tag)s' not defined in '%(file)s'.",
0578 tag=tag, file=aconf.path))
0579
0580
0581 def assert_syntax (aconfs_catpaths, onabortf=None):
0582
0583 checkf = msgfmt(options=["--check"])
0584 numerr = 0
0585 for aconf, catpaths in aconfs_catpaths:
0586 for catpath, acatpath in catpaths:
0587 numerr += checkf(catpath)
0588 if numerr:
0589 if onabortf:
0590 onabortf()
0591 error(_("@info",
0592 "Invalid syntax in some files, see the reports above. "
0593 "Ascription aborted."))
0594 return numerr
0595
0596
0597 def setup_progress (aconfs_catpaths, addfmt):
0598
0599 acps = [y[0] for x in aconfs_catpaths for y in x[1]]
0600 return init_file_progress(acps, addfmt=addfmt)
0601
0602
0603 # Exclusive states of a message, as reported by Message.state().
0604 _st_tran = "T"
0605 _st_fuzzy = "F"
0606 _st_untran = "U"
0607 _st_otran = "OT"
0608 _st_ofuzzy = "OF"
0609 _st_ountran = "OU"
0610 _all_states = (
0611 _st_tran, _st_fuzzy, _st_untran,
0612 _st_otran, _st_ofuzzy, _st_ountran,
0613 )
0614
0615
0616 def status (options, aconfs_catpaths, mode):
0617
0618 # Count ascribed and unascribed messages through catalogs.
0619 counts_a = dict([(x, {}) for x in _all_states])
0620 counts_na = dict([(x, {}) for x in _all_states])
0621
0622 upprog = setup_progress(aconfs_catpaths,
0623 t_("@info:progress",
0624 "Examining state: %(file)s"))
0625 for aconf, catpaths in aconfs_catpaths:
0626 for catpath, acatpath in catpaths:
0627 upprog(catpath)
0628 # Open current and ascription catalog.
0629 cat = Catalog(catpath, monitored=False)
0630 acat = Catalog(acatpath, create=True, monitored=False)
0631 # Count ascribed and non-ascribed by original catalog.
0632 nselected = 0
0633 for msg in cat:
0634 purge_msg(msg)
0635 ahist = collect_ascription_history(
0636 msg, acat, aconf,
0637 hfilter=options.hfilter, addrem=options.addrem, nomrg=True)
0638 if ahist[0].user is None and not has_tracked_parts(msg):
0639 continue # pristine
0640 if not mode.selector(msg, cat, ahist, aconf):
0641 continue # not selected
0642 counts = ahist[0].user is None and counts_na or counts_a
0643 st = msg.state()
0644 if catpath not in counts[st]:
0645 counts[st][catpath] = 0
0646 counts[st][catpath] += 1
0647 nselected += 1
0648 # Count non-ascribed by ascription catalog.
0649 for amsg in acat:
0650 if amsg not in cat:
0651 ast = amsg.state()
0652 st = None
0653 if ast == _st_tran:
0654 st = _st_otran
0655 elif ast == _st_fuzzy:
0656 st = _st_ofuzzy
0657 elif ast == _st_untran:
0658 st = _st_ountran
0659 if st:
0660 if catpath not in counts_na[st]:
0661 counts_na[st][catpath] = 0
0662 counts_na[st][catpath] += 1
0663 # Cancel counts if maximum selection fraction exceeded.
0664 if float(nselected) / len(cat) > options.max_fraction_select:
0665 for counts in (counts_a, counts_na):
0666 for st in _all_states:
0667 if catpath in counts[st]:
0668 counts[st].pop(catpath)
0669 upprog()
0670
0671 # Some general data for tabulation of output.
0672 coln = [_("@title:column translated messages", "msg/t"),
0673 _("@title:column fuzzy messages", "msg/f"),
0674 _("@title:column untranslated messages", "msg/u"),
0675 _("@title:column obsolete translated messages", "msg/ot"),
0676 _("@title:column obsolete fuzzy messages", "msg/of"),
0677 _("@title:column obsolete untranslated messages", "msg/ou")]
0678 none="-"
0679
0680 # NOTE: When reporting, do not show anything if there are
0681 # neither ascribed nor non-ascribed messages selected.
0682 # If there are some ascribed and none non-ascribed,
0683 # show only the row for ascribed.
0684 # However, if there are some non-ascribed but none ascribed,
0685 # still show the row for ascribed, to not accidentally confuse
0686 # non-ascribed for ascribed.
0687
0688 # Report totals.
0689 totals_a, totals_na = {}, {}
0690 for totals, counts in ((totals_a, counts_a), (totals_na, counts_na)):
0691 for st, cnt_per_cat in list(counts.items()):
0692 totals[st] = sum(cnt_per_cat.values())
0693 # See previous NOTE.
0694 if sum(totals_a.values()) > 0 or sum(totals_na.values()) > 0:
0695 rown = [_("@title:row number of ascribed messages",
0696 "ascribed")]
0697 data = [[totals_a[x] or None] for x in _all_states]
0698 if sum(totals_na.values()) > 0:
0699 rown.append(_("@title:row number of unascribed messages",
0700 "unascribed"))
0701 for i in range(len(_all_states)):
0702 data[i].append(totals_na[_all_states[i]] or None)
0703 report(tabulate(data=data, coln=coln, rown=rown,
0704 none=none, colorize=True))
0705
0706 # Report counts per catalog if requested.
0707 if options.show_by_file:
0708 catpaths = set()
0709 for counts in (counts_a, counts_na):
0710 catpaths.update(sum([list(x.keys()) for x in list(counts.values())], []))
0711 catpaths = sorted(catpaths)
0712 if catpaths:
0713 coln.insert(0, _("@title:column", "catalog"))
0714 coln.insert(1, _("@title:column state (asc/nasc)", "st"))
0715 data = [[] for x in _all_states]
0716 for catpath in catpaths:
0717 cc_a = [counts_a[x].get(catpath, 0) for x in _all_states]
0718 cc_na = [counts_na[x].get(catpath, 0) for x in _all_states]
0719 # See previous NOTE.
0720 if sum(cc_a) > 0 or sum(cc_na) > 0:
0721 data[0].append(catpath)
0722 data[1].append(
0723 _("@item:intable number of ascribed messages",
0724 "asc"))
0725 for datac, cc in zip(data[2:], cc_a):
0726 datac.append(cc or None)
0727 if sum(cc_na) > 0:
0728 data[0].append("^^^")
0729 data[1].append(
0730 _("@item:intable number of unascribed messages",
0731 "nasc"))
0732 for datac, cc in zip(data[2:], cc_na):
0733 datac.append(cc or None)
0734 if any(data):
0735 dfmt = ["%%-%ds" % max([len(x) for x in catpaths])]
0736 report("-")
0737 report(tabulate(data=data, coln=coln, dfmt=dfmt,
0738 none=none, colorize=True))
0739
0740
0741 # FIXME: Factor out into message module.
0742 _fields_current = (
0743 "msgctxt", "msgid", "msgid_plural",
0744 )
0745 _fields_previous = (
0746 "msgctxt_previous", "msgid_previous", "msgid_plural_previous",
0747 )
0748
0749 def msg_to_previous (msg, copy=True):
0750
0751 if msg.fuzzy and msg.msgid_previous is not None:
0752 pmsg = MessageUnsafe(msg) if copy else msg
0753 for fcurr, fprev in zip(_fields_current, _fields_previous):
0754 setattr(pmsg, fcurr, pmsg.get(fprev))
0755 pmsg.unfuzzy()
0756 return pmsg
0757
0758
0759 def restore_reviews (aconfs_catpaths, revspecs_by_catmsg):
0760
0761 upprog = setup_progress(aconfs_catpaths,
0762 t_("@info:progress",
0763 "Restoring reviews: %(file)s"))
0764 nrestored = 0
0765 for aconf, catpaths in aconfs_catpaths:
0766 for catpath, acatpath in catpaths:
0767 upprog(catpath)
0768 revels_by_msg = revspecs_by_catmsg.get(catpath)
0769 if revels_by_msg:
0770 cat = Catalog(catpath, monitored=True)
0771 for msgref, revels in sorted(revels_by_msg.items()):
0772 msg = cat[msgref - 1]
0773 revtags, unrevd, revok = revels
0774 restore_review_flags(msg, revtags, unrevd)
0775 nrestored += 1
0776 sync_and_rep(cat, shownmod=False)
0777 if aconf.vcs.is_versioned(acatpath):
0778 aconf.vcs.revert(acatpath)
0779 # ...no else: because revert may cause the file
0780 # not to be versioned any more.
0781 if not aconf.vcs.is_versioned(acatpath):
0782 os.remove(acatpath)
0783
0784 if nrestored > 0:
0785 report(n_("@info:progress",
0786 "===== Review elements restored to %(num)d message.",
0787 "===== Review elements restored to %(num)d messages.",
0788 num=nrestored))
0789
0790
0791 def restore_review_flags (msg, revtags, unrevd):
0792
0793 for tag in revtags:
0794 flag = _revdflags[0]
0795 if tag:
0796 flag += _flagtagsep + tag
0797 msg.flag.add(flag)
0798 if unrevd:
0799 msg.flag.add(_urevdflags[0])
0800
0801 return msg
0802
0803
0804 def commit (options, aconfs_catpaths, mode):
0805
0806 assert_mode_user(aconfs_catpaths, mode)
0807
0808 # Ascribe modifications and reviews.
0809 upprog = setup_progress(aconfs_catpaths,
0810 t_("@info:progress",
0811 "Ascribing: %(file)s"))
0812 revels = {}
0813 counts = dict([(x, [0, 0]) for x in _all_states])
0814 aconfs_catpaths_ascmod = []
0815 aconf_by_catpath = {}
0816 for aconf, catpaths in aconfs_catpaths:
0817 aconfs_catpaths_ascmod.append((aconf, []))
0818 for catpath, acatpath in catpaths:
0819 upprog(catpath)
0820 res = commit_cat(options, aconf, mode.user, catpath, acatpath,
0821 mode.selector)
0822 ccounts, crevels, catmod = res
0823 for st, (nmod, nrev) in list(ccounts.items()):
0824 counts[st][0] += nmod
0825 counts[st][1] += nrev
0826 revels[catpath] = crevels
0827 if catmod:
0828 aconfs_catpaths_ascmod[-1][1].append((catpath, acatpath))
0829 aconf_by_catpath[catpath] = aconf
0830 upprog()
0831
0832 onabortf = lambda: restore_reviews(aconfs_catpaths_ascmod, revels)
0833
0834 # Assert that all reviews were good.
0835 unknown_revtags = []
0836 for catpath, revels1 in sorted(revels.items()):
0837 aconf = aconf_by_catpath[catpath]
0838 for msgref, (revtags, unrevd, revok) in sorted(revels1.items()):
0839 if not revok:
0840 onabortf()
0841 error("Ascription aborted due to earlier warnings.")
0842
0843 assert_syntax(aconfs_catpaths_ascmod, onabortf=onabortf)
0844 # ...must be done after committing, to have all review elements purged
0845
0846 coln = [_("@title:column number of modified messages",
0847 "modified")]
0848 rown = []
0849 data = [[]]
0850 for st, stlabel in (
0851 (_st_tran,
0852 _("@title:row number of translated messages",
0853 "translated")),
0854 (_st_fuzzy,
0855 _("@title:row number of fuzzy messages",
0856 "fuzzy")),
0857 (_st_untran,
0858 _("@title:row number of untranslated messages",
0859 "untranslated")),
0860 (_st_otran,
0861 _("@title:row number of obsolete translated messages",
0862 "obsolete/t")),
0863 (_st_ofuzzy,
0864 _("@title:row number of obsolete fuzzy messages",
0865 "obsolete/f")),
0866 (_st_ountran,
0867 _("@title:row number of obsolete untranslated messages",
0868 "obsolete/u")),
0869 ):
0870 if counts[st][1] > 0 and len(coln) < 2:
0871 coln.append(_("@title:column number of reviewed messages",
0872 "reviewed"))
0873 data.append([])
0874 if counts[st][0] > 0 or counts[st][1] > 0:
0875 rown.append(stlabel)
0876 data[0].append(counts[st][0] or None)
0877 if len(coln) >= 2:
0878 data[1].append(counts[st][1] or None)
0879 if rown:
0880 report(_("@info:progress", "===== Ascription summary:"))
0881 report(tabulate(data, coln=coln, rown=rown, none="-",
0882 colorize=True))
0883
0884 if options.vcs_commit:
0885 vcs_commit_catalogs(aconfs_catpaths, mode.user,
0886 message=options.message, onabortf=onabortf)
0887 # ...not configs_catpaths_ascmod, as non-ascription relevant
0888 # modifications may exist (e.g. new pristine catalog added).
0889
0890
0891 def diff (options, aconfs_catpaths, mode):
0892
0893 upprog = setup_progress(aconfs_catpaths,
0894 t_("@info:progress",
0895 "Diffing for review: %(file)s"))
0896 ndiffed = 0
0897 for aconf, catpaths in aconfs_catpaths:
0898 for catpath, acatpath in catpaths:
0899 upprog(catpath)
0900 ndiffed += diff_cat(options, aconf, catpath, acatpath,
0901 mode.selector, mode.aselector)
0902 upprog()
0903 if ndiffed > 0:
0904 report(n_("@info:progress",
0905 "===== %(num)d message diffed for review.",
0906 "===== %(num)d messages diffed for review.",
0907 num=ndiffed))
0908
0909
0910 def purge (options, aconfs_catpaths, mode):
0911
0912 upprog = setup_progress(aconfs_catpaths,
0913 t_("@info:progress",
0914 "Purging review elements: %(file)s"))
0915 npurged = 0
0916 for aconf, catpaths in aconfs_catpaths:
0917 for catpath, acatpath in catpaths:
0918 upprog(catpath)
0919 npurged += purge_cat(options, aconf, catpath, acatpath,
0920 mode.selector)
0921 upprog()
0922
0923 if npurged > 0:
0924 if not options.keep_flags:
0925 report(n_("@info:progress",
0926 "===== Review elements purged from %(num)d message.",
0927 "===== Review elements purged from %(num)d messages.",
0928 num=npurged))
0929 else:
0930 report(n_("@info:progress",
0931 "===== Review elements purged from %(num)d message "
0932 "(flags kept).",
0933 "===== Review elements purged from %(num)d messages "
0934 "(flags kept).",
0935 num=npurged))
0936
0937 return npurged
0938
0939
0940 def history (options, aconfs_catpaths, mode):
0941
0942 upprog = setup_progress(aconfs_catpaths,
0943 t_("@info:progress",
0944 "Computing histories: %(file)s"))
0945 nshown = 0
0946 for aconf, catpaths in aconfs_catpaths:
0947 for catpath, acatpath in catpaths:
0948 upprog(catpath)
0949 nshown += history_cat(options, aconf, catpath, acatpath,
0950 mode.selector)
0951 upprog()
0952 if nshown > 0:
0953 report(n_("@info:progress",
0954 "===== Histories computed for %(num)d message.",
0955 "===== Histories computed for %(num)d messages.",
0956 num=nshown))
0957
0958
0959 def commit_cat (options, aconf, user, catpath, acatpath, stest):
0960
0961 # Open current catalog and ascription catalog.
0962 # Monitored, for removal of review elements.
0963 cat = Catalog(catpath, monitored=True)
0964 acat = prep_write_asc_cat(acatpath, aconf)
0965
0966 revtags_ovr = None
0967 if options.all_reviewed:
0968 revtags_ovr = options.tags
0969
0970 # Collect unascribed messages, but ignoring pristine ones
0971 # (those which are both untranslated and without history).
0972 # Collect and purge any review elements.
0973 # Check if any modification cannot be due to merging
0974 # (if header update is requested).
0975 mod_msgs = []
0976 rev_msgs = []
0977 revels_by_msg = {}
0978 counts = dict([(x, [0, 0]) for x in _all_states])
0979 counts0 = counts.copy()
0980 any_nonmerges = False
0981 prev_msgs = []
0982 check_mid_msgs = []
0983 for msg in cat:
0984 mod, revtags, unrevd = purge_msg(msg)
0985 if mod:
0986 revels_by_msg[msg.refentry] = [revtags, unrevd, True]
0987 ahist = collect_ascription_history(msg, acat, aconf) # after purging
0988 # Do not ascribe anything if the message is new and untranslated.
0989 if ( ahist[0].user is None and len(ahist) == 1
0990 and not has_tracked_parts(msg)
0991 ):
0992 continue
0993 # Possibly ascribe review only if the message passes the selector.
0994 if stest(msg, cat, ahist, aconf) and (mod or revtags_ovr):
0995 if revtags_ovr:
0996 revtags = revtags_ovr
0997 unrevd = False
0998 if revtags and not unrevd: # unreviewed flag overrides
0999 rev_msgs.append((msg, revtags))
1000 counts[msg.state()][1] += 1
1001 # Check and record if review tags are not valid.
1002 unknown_revtags = revtags.difference(aconf.revtags)
1003 if unknown_revtags:
1004 revels_by_msg[msg.refentry][-1] = False
1005 tagfmt = format_item_list(sorted(unknown_revtags))
1006 warning_on_msg(_("@info",
1007 "Unknown review tags: %(taglist)s.",
1008 taglist=tagfmt), msg, cat)
1009 # Ascribe modification regardless of the selector.
1010 if ahist[0].user is None:
1011 mod_msgs.append(msg)
1012 counts[msg.state()][0] += 1
1013 if options.update_headers and not any_nonmerges:
1014 if len(ahist) == 1 or not merge_modified(ahist[1].msg, msg):
1015 any_nonmerges = True
1016 # Record that reconstruction of the post-merge message
1017 # should be tried if this message has no prior history
1018 # but it is not pristine (it may be that the translator
1019 # has merged the catalog and updated fuzzy messages in one step,
1020 # without committing the catalog right after merging).
1021 if len(ahist) == 1:
1022 check_mid_msgs.append(msg)
1023 # Collect latest historical version of the message,
1024 # in case reconstruction of post-merge messages is needed.
1025 if ahist[0].user is not None or len(ahist) > 1:
1026 pmsg = ahist[1 if ahist[0].user is None else 0].msg
1027 prev_msgs.append(pmsg)
1028
1029 # Collect non-obsolete ascribed messages that no longer have
1030 # original counterpart, to ascribe as obsolete.
1031 # If reconstruction of post-merge messages is needed,
1032 # also collect latest historical versions.
1033 cat.sync_map() # in case key fields were purged
1034 for amsg in acat:
1035 if amsg not in cat:
1036 ast = amsg.state()
1037 st = None
1038 if ast == _st_tran:
1039 st = _st_otran
1040 elif ast == _st_fuzzy:
1041 st = _st_ofuzzy
1042 elif ast == _st_untran:
1043 st = _st_ountran
1044 if st or check_mid_msgs:
1045 msg = collect_ascription_history_segment(amsg, acat, aconf)[0].msg
1046 if check_mid_msgs:
1047 prev_msgs.append(msg)
1048 if st:
1049 msg.obsolete = True
1050 mod_msgs.append(msg)
1051 counts[st][0] += 1
1052
1053 # Shortcut if nothing to do, because sync_and_rep later are expensive.
1054 if not mod_msgs and not revels_by_msg:
1055 # No messages to commit.
1056 return counts0, revels_by_msg, False
1057
1058 # Construct post-merge messages.
1059 mod_mid_msgs = []
1060 if check_mid_msgs and not acat.created():
1061 mid_cat = create_post_merge_cat(cat, prev_msgs)
1062 for msg in check_mid_msgs:
1063 mid_msg = mid_cat.get(msg)
1064 if ( mid_msg is not None
1065 and mid_msg.fuzzy
1066 and not ascription_equal(mid_msg, msg)
1067 ):
1068 mod_mid_msgs.append(mid_msg)
1069
1070 # Ascribe modifications.
1071 for mid_msg in mod_mid_msgs: # ascribe post-merge before actual
1072 ascribe_modification(mid_msg, user, _dt_start, acat, aconf)
1073 for msg in mod_msgs:
1074 ascribe_modification(msg, user, _dt_start, acat, aconf)
1075
1076 # Ascribe reviews.
1077 for msg, revtags in rev_msgs:
1078 ascribe_review(msg, user, _dt_start, revtags, acat, aconf)
1079
1080 # Update header if requested and translator's modifications detected.
1081 if options.update_headers and any_nonmerges:
1082 cat.update_header(project=cat.name,
1083 title=aconf.title,
1084 name=aconf.users[user].name,
1085 email=aconf.users[user].email,
1086 teamemail=aconf.teamemail,
1087 langname=aconf.langteam,
1088 langcode=aconf.langcode,
1089 plforms=aconf.plforms)
1090
1091 nmod = [len(mod_msgs)]
1092 if len(rev_msgs) > 0:
1093 nmod.append(len(rev_msgs))
1094 catmod = False
1095 if sync_and_rep(cat, nmod=nmod):
1096 catmod = True
1097 if asc_sync_and_rep(acat, shownmod=False, nmod=[0]):
1098 catmod = True
1099
1100 return counts, revels_by_msg, catmod
1101
1102
1103 def diff_cat (options, aconf, catpath, acatpath, stest, aselect):
1104
1105 cat = Catalog(catpath, monitored=True)
1106 acat = Catalog(acatpath, create=True, monitored=False)
1107
1108 # Select messages for diffing.
1109 msgs_to_diff = []
1110 for msg in cat:
1111 purge_msg(msg)
1112 ahist = collect_ascription_history(
1113 msg, acat, aconf,
1114 hfilter=options.hfilter, addrem=options.addrem, nomrg=True)
1115 # Makes no sense to review pristine messages.
1116 if ahist[0].user is None and not has_tracked_parts(msg):
1117 continue
1118 sres = stest(msg, cat, ahist, aconf)
1119 if not sres:
1120 continue
1121 msgs_to_diff.append((msg, ahist, sres))
1122
1123 # Cancel selection if maximum fraction exceeded.
1124 if float(len(msgs_to_diff)) / len(cat) > options.max_fraction_select:
1125 msgs_to_diff = []
1126
1127 if not msgs_to_diff:
1128 return 0
1129
1130 # Diff selected messages.
1131 diffed_msgs = []
1132 tagfmt = _flagtagsep.join(options.tags)
1133 for msg, ahist, sres in msgs_to_diff:
1134
1135 # Try to select ascription to differentiate from.
1136 # (Note that ascription indices returned by selectors are 1-based.)
1137 i_asc = None
1138 if aselect:
1139 asres = aselect(msg, cat, ahist, aconf)
1140 i_asc = (asres - 1) if asres else None
1141 elif not isinstance(sres, bool):
1142 # If there is no ascription selector, but basic selector returned
1143 # an ascription index, use first earlier non-fuzzy for diffing.
1144 i_asc = sres - 1
1145 i_asc = first_non_fuzzy(ahist, i_asc + 1)
1146
1147 # Differentiate and flag.
1148 amsg = i_asc is not None and ahist[i_asc].msg or None
1149 if amsg is not None:
1150 if editprob(amsg.msgid, msg.msgid) > options.min_adjsim_diff:
1151 msg_ediff(amsg, msg, emsg=msg, pfilter=options.sfilter)
1152 flag = _diffflag
1153 else:
1154 # If to great difference, add special flag and do not diff.
1155 flag = _diffflag_ign
1156 else:
1157 # If no previous ascription selected, add special flag.
1158 flag = _diffflag_tot
1159 if tagfmt:
1160 flag += _flagtagsep + tagfmt
1161 msg.flag.add(flag)
1162
1163 # Add ascription chain comment.
1164 ascfmts = []
1165 i_from = (i_asc - 1) if i_asc is not None else len(ahist) - 1
1166 for i in range(i_from, -1, -1):
1167 a = ahist[i]
1168 shtype = {AscPoint.ATYPE_MOD: "m",
1169 AscPoint.ATYPE_REV: "r"}[a.type]
1170 if a.tag:
1171 ascfmt = "%s:%s(%s)" % (a.user, shtype, a.tag)
1172 else:
1173 ascfmt = "%s:%s" % (a.user, shtype)
1174 ascfmts.append(ascfmt)
1175 achnfmt = "%s %s" % (_achncmnt, " ".join(ascfmts))
1176 msg.auto_comment.append(achnfmt)
1177
1178 diffed_msgs.append(msg)
1179
1180 sync_and_rep(cat)
1181
1182 # Open in the PO editor if requested.
1183 if options.po_editor:
1184 for msg in diffed_msgs:
1185 options.po_editor(msg, cat,
1186 report=_("@info note on selected message",
1187 "Selected for review."))
1188
1189 return len(diffed_msgs)
1190
1191
1192 _subreflags = "|".join(_all_flags)
1193 _subrecmnts = "|".join(_all_cmnts)
1194 _any_to_purge_rx = re.compile(r"^\s*(#,.*\b(%s)|#\.\s*(%s))"
1195 % (_subreflags, _subrecmnts),
1196 re.M|re.U)
1197
1198 # Quickly check if it may be that some messages in the PO file
1199 # have review elements (diffs, flags).
1200 def may_have_revels (catpath):
1201
1202 return bool(_any_to_purge_rx.search(open(catpath).read()))
1203
1204
1205 def purge_cat (options, aconf, catpath, acatpath, stest):
1206
1207 if not may_have_revels(catpath):
1208 return 0
1209
1210 cat = Catalog(catpath, monitored=True)
1211 acat = Catalog(acatpath, create=True, monitored=False)
1212
1213 # Select messages to purge.
1214 msgs_to_purge = []
1215 for msg in cat:
1216 cmsg = MessageUnsafe(msg)
1217 purge_msg(cmsg)
1218 ahist = collect_ascription_history(
1219 cmsg, acat, aconf,
1220 hfilter=options.hfilter, addrem=options.addrem, nomrg=True)
1221 if not stest(cmsg, cat, ahist, aconf):
1222 continue
1223 msgs_to_purge.append(msg)
1224
1225 # Does observing options.max_fraction_select makes sense for purging?
1226 ## Cancel selection if maximum fraction exceeded.
1227 #if float(len(msgs_to_purge)) / len(cat) > options.max_fraction_select:
1228 #msgs_to_purge = []
1229
1230 # Purge selected messages.
1231 npurged = 0
1232 for msg in msgs_to_purge:
1233 res = purge_msg(msg, keepflags=options.keep_flags)
1234 mod, revtags, unrevd = res
1235 if mod:
1236 npurged += 1
1237
1238 sync_and_rep(cat)
1239
1240 return npurged
1241
1242
1243 def history_cat (options, aconf, catpath, acatpath, stest):
1244
1245 cat = Catalog(catpath, monitored=False)
1246 acat = Catalog(acatpath, create=True, monitored=False)
1247
1248 # Select messages for which to compute histories.
1249 msgs_to_hist = []
1250 for msg in cat:
1251 purge_msg(msg)
1252 ahist = collect_ascription_history(
1253 msg, acat, aconf,
1254 hfilter=options.hfilter, addrem=options.addrem, nomrg=True)
1255 if not stest(msg, cat, ahist, aconf):
1256 continue
1257 msgs_to_hist.append((msg, ahist))
1258
1259 # Cancel selection if maximum fraction exceeded.
1260 if float(len(msgs_to_hist)) / len(cat) > options.max_fraction_select:
1261 msgs_to_hist = []
1262
1263 # Compute histories for selected messages.
1264 for msg, ahist in msgs_to_hist:
1265
1266 unasc = ahist[0].user is None
1267 if unasc:
1268 ahist.pop(0)
1269
1270 hlevels = len(ahist)
1271 if options.depth is not None:
1272 hlevels = int(options.depth)
1273 if ahist[0].user is None:
1274 hlevels += 1
1275 if hlevels > len(ahist):
1276 hlevels = len(ahist)
1277
1278 hinfo = []
1279 if hlevels > 0:
1280 hinfo += [_("@info:progress",
1281 "<green>>>> History follows:</green>")]
1282 hfmt = "%%%dd" % len(str(hlevels))
1283 for i in range(hlevels):
1284 a = ahist[i]
1285 if a.type == AscPoint.ATYPE_MOD:
1286 anote = _("@item:intable",
1287 "<bold>#%(pos)d</bold> "
1288 "modified by %(user)s on %(date)s",
1289 pos=a.pos, user=a.user, date=a.date)
1290 elif a.type == AscPoint.ATYPE_REV:
1291 if not a.tag:
1292 anote = _("@item:intable",
1293 "<bold>#%(pos)d</bold> "
1294 "reviewed by %(user)s on %(date)s",
1295 pos=a.pos, user=a.user, date=a.date)
1296 else:
1297 anote = _("@item:intable",
1298 "<bold>#%(pos)d</bold> "
1299 "reviewed (%(tag)s) by %(user)s on %(date)s",
1300 pos=a.pos, user=a.user, tag=a.tag, date=a.date)
1301 else:
1302 warning_on_msg(
1303 _("@info",
1304 "Unknown ascription type '%(type)s' found in history.",
1305 type=a.type), msg, cat)
1306 continue
1307 hinfo += [anote]
1308 if not a.type == AscPoint.ATYPE_MOD:
1309 # Nothing more to show if this ascription is not modification.
1310 continue
1311 i_next = i + 1
1312 if i_next == len(ahist):
1313 # Nothing more to show at end of history.
1314 continue
1315 dmsg = MessageUnsafe(a.msg)
1316 nmsg = ahist[i_next].msg
1317 if dmsg != nmsg:
1318 msg_ediff(nmsg, dmsg, emsg=dmsg,
1319 pfilter=options.sfilter, colorize=True)
1320 dmsgfmt = dmsg.to_string(force=True,
1321 wrapf=cat.wrapf()).rstrip("\n")
1322 hindent = " " * (len(hfmt % 0) + 2)
1323 hinfo += [hindent + x for x in dmsgfmt.split("\n")]
1324 hinfo = cjoin(hinfo, "\n")
1325
1326 if unasc or msg.fuzzy:
1327 pmsg = None
1328 i_nfasc = first_non_fuzzy(ahist)
1329 if i_nfasc is not None:
1330 pmsg = ahist[i_nfasc].msg
1331 elif msg.fuzzy and msg.msgid_previous is not None:
1332 pmsg = msg_to_previous(msg)
1333 if pmsg is not None:
1334 for fprev in _fields_previous:
1335 setattr(msg, fprev, None)
1336 msg_ediff(pmsg, msg, emsg=msg,
1337 pfilter=options.sfilter, colorize=True)
1338 report_msg_content(msg, cat,
1339 note=(hinfo or None), delim=("-" * 20))
1340
1341 return len(msgs_to_hist)
1342
1343
1344 _revflags_rx = re.compile(r"^(%s)(?: */(.*))?" % "|".join(_all_flags), re.I)
1345
1346 def purge_msg (msg, keepflags=False):
1347
1348 modified = False
1349
1350 # Remove review flags.
1351 diffed = False
1352 revtags = set()
1353 unrevd = False
1354 for flag in list(msg.flag): # modified inside
1355 m = _revflags_rx.search(flag)
1356 if m:
1357 sflag = m.group(1)
1358 tagstr = m.group(2) or ""
1359 tags = [x.strip() for x in tagstr.split(_flagtagsep)]
1360 if sflag not in _urevdflags:
1361 revtags.update(tags)
1362 if sflag == _diffflag:
1363 # ...must not check ...in _diffflags because with
1364 # those other flags there is actually no diff.
1365 diffed = True
1366 else:
1367 unrevd = True
1368 msg.flag.remove(flag)
1369 modified = True
1370
1371 # Remove review comments.
1372 i = 0
1373 while i < len(msg.auto_comment):
1374 cmnt = msg.auto_comment[i].strip()
1375 if cmnt.startswith(_all_cmnts):
1376 msg.auto_comment.pop(i)
1377 modified = True
1378 else:
1379 i += 1
1380
1381 # Remove any leftover previous fields.
1382 if msg.translated:
1383 for fprev in _fields_previous:
1384 if msg.get(fprev) is not None:
1385 setattr(msg, fprev, None)
1386 modified = True
1387
1388 if diffed:
1389 msg_ediff_to_new(msg, rmsg=msg)
1390 if keepflags:
1391 restore_review_flags(msg, revtags, unrevd)
1392
1393 return modified, revtags, unrevd
1394
1395
1396 def prep_write_asc_cat (acatpath, aconf):
1397
1398 if not os.path.isfile(acatpath):
1399 return init_asc_cat(acatpath, aconf)
1400 else:
1401 return Catalog(acatpath, monitored=True, wrapping=_ascwrapping)
1402
1403
1404 def init_asc_cat (acatpath, aconf):
1405
1406 acat = Catalog(acatpath, create=True, monitored=True, wrapping=_ascwrapping)
1407 ahdr = acat.header
1408
1409 ahdr.title = Monlist(["Ascription shadow for %s.po" % acat.name])
1410
1411 translator = "Ascriber"
1412
1413 if aconf.teamemail:
1414 author = "%s <%s>" % (translator, aconf.teamemail)
1415 else:
1416 author = "%s" % translator
1417 ahdr.author = Monlist([author])
1418
1419 ahdr.copyright = "Copyright same as for the original catalog."
1420 ahdr.license = "License same as for the original catalog."
1421 ahdr.comment = Monlist(["===== DO NOT EDIT MANUALLY ====="])
1422
1423 ahdr.set_field("Project-Id-Version", str(acat.name))
1424 ahdr.set_field("Report-Msgid-Bugs-To", str(aconf.teamemail or ""))
1425 ahdr.set_field("PO-Revision-Date", format_datetime(_dt_start))
1426 ahdr.set_field("Content-Type", "text/plain; charset=UTF-8")
1427 ahdr.set_field("Content-Transfer-Encoding", "8bit")
1428
1429 if aconf.teamemail:
1430 ltr = "%s <%s>" % (translator, aconf.teamemail)
1431 else:
1432 ltr = translator
1433 ahdr.set_field("Last-Translator", str(ltr))
1434
1435 if aconf.langteam:
1436 if aconf.teamemail:
1437 tline = "%s <%s>" % (aconf.langteam, aconf.teamemail)
1438 else:
1439 tline = aconf.langteam
1440 ahdr.set_field("Language-Team", str(tline))
1441 else:
1442 ahdr.remove_field("Language-Team")
1443
1444 if aconf.langcode:
1445 ahdr.set_field("Language", str(aconf.langcode))
1446 else:
1447 ahdr.remove_field("Language")
1448
1449 if aconf.plforms:
1450 ahdr.set_field("Plural-Forms", str(aconf.plforms))
1451 else:
1452 ahdr.remove_field("Plural-Forms")
1453
1454 return acat
1455
1456
1457 def update_asc_hdr (acat):
1458
1459 acat.header.set_field("PO-Revision-Date", format_datetime(_dt_start))
1460
1461
1462 def create_post_merge_cat (cat, prev_msgs):
1463
1464 # Prepare previous catalog based on ascription catalog.
1465 prev_cat = Catalog("", create=True, monitored=False)
1466 prev_cat.header = Header(cat.header)
1467 for prev_msg in prev_msgs:
1468 prev_cat.add_last(prev_msg)
1469 tmpf1 = NamedTemporaryFile(prefix="pology-merged-", suffix=".po")
1470 prev_cat.filename = tmpf1.name
1471 prev_cat.sync()
1472
1473 # Prepare template based on current catalog.
1474 tmpl_cat = Catalog("", create=True, monitored=False)
1475 tmpl_cat.header = Header(cat.header)
1476 for msg in cat:
1477 if not msg.obsolete:
1478 tmpl_msg = MessageUnsafe(msg)
1479 tmpl_msg.clear()
1480 tmpl_cat.add_last(tmpl_msg)
1481 tmpf2 = NamedTemporaryFile(prefix="pology-template-", suffix=".pot")
1482 tmpl_cat.filename = tmpf2.name
1483 tmpl_cat.sync()
1484
1485 # Merge previous catalog using current catalog as template.
1486 mid_cat = merge_pofile(prev_cat.filename, tmpl_cat.filename,
1487 getcat=True, monitored=False, quiet=True)
1488
1489 return mid_cat
1490
1491
1492 _modified_cats = []
1493
1494 def sync_and_rep (cat, shownmod=True, nmod=None):
1495
1496 if shownmod and nmod is None:
1497 nmod = [0]
1498 for msg in cat:
1499 if msg.modcount:
1500 nmod[0] += 1
1501
1502 modified = cat.sync()
1503 if nmod and sum(nmod) > 0: # DO NOT check instead modified == True
1504 if shownmod:
1505 nmodfmt = "/".join("%d" % x for x in nmod)
1506 report("%s (%s)" % (cat.filename, nmodfmt))
1507 else:
1508 report("%s" % cat.filename)
1509 _modified_cats.append(cat.filename)
1510
1511 return modified
1512
1513
1514 def asc_sync_and_rep (acat, shownmod=True, nmod=None):
1515
1516 if acat.modcount:
1517 update_asc_hdr(acat)
1518 mkdirpath(os.path.dirname(acat.filename))
1519
1520 return sync_and_rep(acat, shownmod=shownmod, nmod=nmod)
1521
1522
1523 # -----------------------------------------------------------------------------
1524
1525 if __name__ == "__main__":
1526 exit_on_exception(main)