File indexing completed on 2024-10-27 11:34:17
0001 # -*- coding: UTF-8 -* 0002 0003 """ 0004 Process ascription configurations, catalogs, and histories. 0005 0006 @note: For the moment, this module is only for internal use within Pology. 0007 Interfaces may change arbitrarily between any two Pology releases. 0008 0009 @author: Chusslove Illich (Часлав Илић) <caslav.ilic@gmx.net> 0010 @license: GPLv3 0011 """ 0012 0013 import codecs 0014 from configparser import SafeConfigParser 0015 import datetime 0016 import os 0017 import re 0018 import time 0019 from types import ModuleType 0020 0021 from pology import PologyError, _, n_, t_ 0022 from pology.header import format_datetime, parse_datetime 0023 from pology.message import Message, MessageUnsafe 0024 from pology.comments import parse_summit_branches 0025 from pology.diff import msg_ediff 0026 from pology.fsops import join_ncwd, str_to_unicode, unicode_to_str 0027 from pology.match import make_msg_fmatcher 0028 from pology.monitored import Monlist 0029 from pology.msgreport import warning_on_msg 0030 from pology.report import warning 0031 from pology.vcs import make_vcs 0032 0033 0034 # ----------------------------------------------------------------------------- 0035 # Ascription data representations. 0036 0037 class AscConfig (object): 0038 """ 0039 Representation of an ascription configuration file. 0040 0041 The settings are reached through class attributes. 0042 Some attributes are raw data read from configuration fields, 0043 while other may be derived based on configuration fields. 0044 0045 Parameters which have "for header updates" in their description 0046 are used for creating and updating ascription catalog headers, 0047 as well as original catalog headers when header update on commit 0048 is requested. They may contain a number of interpolations, see 0049 L{Catalog.update_header<pology.catalog.Catalog.update_header>}. 0050 0051 @ivar path: the path to the ascription configuration file 0052 @type path: string 0053 @ivar catroot: the path to root directory of original catalogs 0054 @type catroot: string 0055 @ivar ascroot: the path to root directory of ascription catalogs 0056 @type ascroot: string 0057 @ivar title: the header title comment (for header updates; 0058 only for original catalogs) 0059 @type title: string or None 0060 @ivar langteam: the language team name (for header updates) 0061 @type langteam: string or None 0062 @ivar teamemail: the language team email address (for header updates) 0063 @type teamemail: string or None 0064 @ivar langcode: the language code (for header updates) 0065 @type langcode: string or None 0066 @ivar plforms: the PO plural forms specification (for header updates) 0067 @type plforms: string or None 0068 @ivar vcs: the version control system for catalogs 0069 @type vcs: L{VcsBase<pology.vcs.VcsBase>} 0070 @ivar commitmsg: the automatic commit message 0071 @type commitmsg: string or None 0072 @ivar revtags: known review tags (empty string always included) 0073 @type revtags: set(string*) 0074 @ivar users: data for ascription users by username 0075 @type users: {string: L{AscUser}*} 0076 """ 0077 0078 def __init__ (self, cfgpath): 0079 """ 0080 Constructor. 0081 0082 Reads the ascription configuration file to set raw and derived 0083 ascription settings. 0084 0085 @param cfgpath: the path to ascription configuration file 0086 @type cfgpath: string 0087 """ 0088 0089 config = SafeConfigParser() 0090 ifl = codecs.open(cfgpath, "r", "UTF-8") 0091 config.readfp(ifl) 0092 ifl.close() 0093 0094 self.path = cfgpath 0095 0096 gsect = dict(config.items("global")) 0097 cpathdir = os.path.dirname(cfgpath) 0098 self.catroot = join_ncwd(cpathdir, gsect.get("catalog-root", "")) 0099 self.ascroot = join_ncwd(cpathdir, gsect.get("ascript-root", "")) 0100 if self.catroot == self.ascroot: 0101 raise PologyError( 0102 _("@info", 0103 "Catalog root and ascription root for '%(file)s' " 0104 "resolve to same path '%(dir)s'.", 0105 file=cfgpath, dir=self.catroot)) 0106 0107 self.title = gsect.get("title", None) 0108 self.langteam = gsect.get("language-team", None) 0109 self.teamemail = gsect.get("team-email", None) 0110 self.langcode = gsect.get("language", None) 0111 self.plforms = gsect.get("plural-header", None) 0112 0113 self.vcs = make_vcs(gsect.get("version-control", "noop")) 0114 0115 self.commitmsg = gsect.get("commit-message", None) 0116 0117 cval = gsect.get("review-tags", None) 0118 if cval is not None: 0119 self.revtags = set(cval.split()) 0120 else: 0121 self.revtags = set() 0122 self.revtags.add("") 0123 0124 self.users = {} 0125 userst = "user-" 0126 for section in config.sections(): 0127 if section.startswith(userst): 0128 user = section[len(userst):] 0129 usect = dict(config.items(section)) 0130 if user in self.users: 0131 raise PologyError( 0132 _("@info", 0133 "Repeated user '%(user)s' in '%(file)s'.", 0134 user=user, file=cpath)) 0135 if "name" not in usect: 0136 raise PologyError( 0137 _("@info", 0138 "The name is missing for " 0139 "user '%(user)s' in '%(file)s'.", 0140 user=user, file=cpath)) 0141 udat = AscUser() 0142 udat.name = usect.get("name") 0143 udat.oname = usect.get("original-name") 0144 udat.email = usect.get("email") 0145 self.users[user] = udat 0146 0147 0148 class AscUser (object): 0149 """ 0150 Representation of an ascription user. 0151 0152 @ivar name: user's name readable in English 0153 @type name: string or None 0154 @ivar oname: user's name in user's native language 0155 @type oname: string or None 0156 @ivar email: user's email address 0157 @type email: string or None 0158 """ 0159 0160 def __init__ (self, name=None, oname=None, email=None): 0161 """ 0162 Constructor. 0163 0164 See attribute documentation for details on parameters. 0165 """ 0166 0167 self.name = name 0168 self.oname = oname 0169 self.email = email 0170 0171 0172 class AscPoint (object): 0173 """ 0174 Representation of an ascription point. 0175 0176 @ivar msg: a stripped version of the original PO message as it appeared 0177 when the ascription was made, containing only 0178 L{extraction-invariant parts<message.Message_base.inv>} 0179 @type msg: L{MessageUnsafe<message.MessageUnsafe>} 0180 @ivar rmsg: the message in the ascription catalog from which 0181 C{msg} was parsed 0182 @type rmsg: L{MessageUnsafe<message.MessageUnsafe>} 0183 @ivar user: the user to whom the ascription was made 0184 @type user: string 0185 @ivar type: the ascription type (one of C{ATYPE_} constants) 0186 @type type: string 0187 @ivar tag: the review tag (from the set defined in ascription config) 0188 @type tag: string 0189 @ivar date: the date when the ascription was made 0190 @type date: datetime.datetime 0191 @ivar slen: the length of the separator in ascription message (C{rmsg}) 0192 @type slen: int 0193 @ivar fuzz: whether the original message was fuzzy 0194 @type fuzz: bool 0195 @ivar obs: whether the original message was obsolete 0196 @type obs: bool 0197 @ivar pos: the position of this ascription point within 0198 the ascription history (increasing from 1, 1 is the latest by date) 0199 @type pos: int 0200 """ 0201 0202 _known_attrs = ( 0203 "rmsg", "msg", 0204 "user", "type", ("tag", ""), "date", 0205 "slen", "fuzz", "obs", 0206 "pos" 0207 ) 0208 0209 def __init__ (self, apoint=None): 0210 """ 0211 Create an empty ascription point or a shallow copy of another. 0212 0213 @param apoint: an ascription point 0214 @type apoint: L{AscPoint} 0215 """ 0216 0217 for attr in AscPoint._known_attrs: 0218 if isinstance(attr, tuple): 0219 attr, dval = attr 0220 else: 0221 attr, dval = attr, None 0222 if apoint is not None: 0223 self.__dict__[attr] = apoint.__dict__[attr] 0224 else: 0225 self.__dict__[attr] = dval 0226 0227 0228 # Ascription types. 0229 # NOTE: These string are written into and read from ascription files. 0230 ATYPE_MOD = "modified" 0231 ATYPE_REV = "reviewed" 0232 0233 0234 # ----------------------------------------------------------------------------- 0235 # Collecting ascription configurations and catalog paths. 0236 0237 0238 def collect_ascription_associations (catpaths): 0239 """ 0240 Build up ascription associations for catalog paths. 0241 0242 For each catalog path, the ascription configuration to which it 0243 belongs is found and parsed, and the corresponding ascription 0244 catalog path assembled. 0245 The association is organized as list of two-tuples; 0246 the first element is the parsed ascription configuration, 0247 and the second element the list of two-tuples of original 0248 catalog paths and associated ascription catalog paths 0249 (whether the ascription catalog already exists or not). 0250 For example, if the input is:: 0251 0252 ["foo/alpha.po", "foo/bravo.po", "bar/november.po"] 0253 0254 and the files are covered by ascription configurations at 0255 C{foo/ascription-config} and C{bar/ascription-config}, 0256 the return value is:: 0257 0258 [(AscConfig("foo/ascription-config"), 0259 [("foo/alpha.po", "foo-ascript/alpha.po"), 0260 ("foo/bravo.po", "foo-ascript/bravo.po")]), 0261 (AscConfig("bar/ascription-config"), 0262 [("bar/november.po", "bar-ascript/november.po")])] 0263 0264 (assuming that both ascription configurations set C{*-ascript/} 0265 directories as corresponding ascription catalog roots). 0266 0267 @param catpaths: a list of catalog paths 0268 @type catpaths: [string*] 0269 @returns: the ascription association list 0270 @rtype: [(AscConfig, [(string, string)*])*] 0271 """ 0272 0273 aconfs_by_cfgpath = {} 0274 catpaths_by_cfgpath = {} 0275 for catpath in catpaths: 0276 # Look for the first config file up the directory tree. 0277 parent = os.path.dirname(os.path.abspath(catpath)) 0278 cfgpath = None 0279 while True: 0280 for cfgname in ("ascription-config", "ascribe"): 0281 test_cfgpath = os.path.join(parent, cfgname) 0282 if os.path.isfile(test_cfgpath): 0283 cfgpath = test_cfgpath 0284 break 0285 if cfgpath: 0286 break 0287 pparent = parent 0288 parent = os.path.dirname(parent) 0289 if parent == pparent: 0290 break 0291 if not cfgpath: 0292 raise PologyError( 0293 _("@info", 0294 "Cannot find ascription configuration for '%(file)s'.", 0295 file=catpath)) 0296 cfgpath = join_ncwd(cfgpath) # for nicer message output 0297 aconf = aconfs_by_cfgpath.get(cfgpath) 0298 if not aconf: 0299 # New config, load. 0300 aconf = AscConfig(cfgpath) 0301 aconfs_by_cfgpath[cfgpath] = aconf 0302 catpaths_by_cfgpath[cfgpath] = [] 0303 catpaths = catpaths_by_cfgpath.get(cfgpath) 0304 0305 # If this catalog is under ascription, 0306 # determine path to ascription catalog. 0307 # Ignore it otherwise. 0308 relcatpath = _relpath(catpath, aconf.catroot) 0309 if relcatpath is not None: 0310 acatpath = join_ncwd(aconf.ascroot, relcatpath) 0311 catpath = join_ncwd(catpath) 0312 catpaths.append((catpath, acatpath)) 0313 0314 # Link config objects and catalog paths. 0315 aconfs_catpaths = [] 0316 for cfgpath in sorted(aconfs_by_cfgpath): 0317 aconfs_catpaths.append((aconfs_by_cfgpath[cfgpath], 0318 catpaths_by_cfgpath[cfgpath])) 0319 0320 return aconfs_catpaths 0321 0322 0323 def _relpath (path, dirpath): 0324 0325 absdirpath = os.path.abspath(dirpath) 0326 lenadpath = len(absdirpath) 0327 lenadpathws = lenadpath + len(os.path.sep) 0328 abspath = os.path.abspath(path) 0329 p = abspath.find(absdirpath) 0330 if p == 0 and abspath[lenadpath:lenadpathws] == os.path.sep: 0331 return abspath[lenadpathws:] 0332 else: 0333 return None 0334 0335 0336 # ----------------------------------------------------------------------------- 0337 # Reading ascriptions. 0338 0339 # FIXME: Factor out into message module. 0340 _id_fields = ( 0341 "msgctxt", "msgid", 0342 ) 0343 _nonid_fields = ( 0344 "msgid_plural", "msgstr", 0345 ) 0346 _fields_previous = ( 0347 "msgctxt_previous", "msgid_previous", "msgid_plural_previous", 0348 ) 0349 _fields_current = ( 0350 "msgctxt", "msgid", "msgid_plural", 0351 ) 0352 _fields_comment = ( 0353 "manual_comment", "auto_comment", 0354 ) 0355 _multiple_fields = (() 0356 + ("msgstr",) 0357 + _fields_comment 0358 ) 0359 _nonid_fields_eq_nonfuzzy = (() 0360 + _nonid_fields 0361 + ("manual_comment",) 0362 ) 0363 _nonid_fields_eq_fuzzy = (() 0364 + _nonid_fields_eq_nonfuzzy 0365 + _fields_previous 0366 ) 0367 _translator_parts = ( 0368 "manual_comment", "fuzzy", "msgstr", 0369 ) 0370 0371 # FIXME: ...but this stays here. 0372 _nonid_fields_tracked = (() 0373 + _nonid_fields 0374 + _fields_previous 0375 + ("manual_comment",) 0376 ) 0377 0378 0379 def collect_ascription_history (msg, acat, aconf, 0380 nomrg=False, hfilter=None, shallow=False, 0381 addrem=None): 0382 """ 0383 Collect ascription history of a message. 0384 0385 The ascription history of C{msg} is collected from 0386 the ascription catalog C{acat}, 0387 falling under the ascription configuration C{aconf}. 0388 The ascription history is a list of L{AscPoint} objects, 0389 ordered from the newest to the oldest by date of ascription. 0390 0391 Some ascription points may be due to merging with template, 0392 when the ascriptions on a catalog were made just after merging. 0393 In many cases of examining the history these ascriptions are not useful, 0394 so they can be removed by setting C{nomrg} to C{True}. 0395 0396 Sometimes it may be convenient to operate on history in which 0397 the translations of historical messages have been filtered, 0398 and this filter can be specified with C{hfilter}. 0399 If under filter two consecutive historical messages become equal, 0400 one of them will be eliminated from the history. 0401 0402 History normally extends in the past through merging with templates 0403 (think of a paragraph-length message in which only one word was changed), 0404 so it may contain messages with keys different from the current message 0405 from some point and onwards. If only the history up to the earliest 0406 message with equal key is desired, C{shallow} can be set to C{True}. 0407 0408 Sometimes it may be convenient to operate on I{incremental} history, 0409 in which every historical message is actually a partial difference 0410 (added, removed or equal segments) from the previous historical message. 0411 This can be requested by setting C{addrem} to one of the values 0412 as described in L{msg_diff<diff.msg_diff>} function. 0413 0414 @param msg: the message from the original catalog 0415 @type msg: L{Message_base<message.Message_base>} 0416 @param acat: the ascription catalog corresponding to the original catalog 0417 @type acat: L{Catalog<catalog.Catalog>} 0418 @param aconf: the ascription configuration which covers the catalogs 0419 @type aconf: L{AscConfig} 0420 @param nomrg: whether to eliminate from history pure merge ascriptions 0421 @type nomrg: bool 0422 @param hfilter: the filter to apply to C{msgstr} fields of 0423 historical messages 0424 @type hfilter: (string)->string 0425 @param shallow: whether to collect history only up to 0426 last historical message with same key 0427 @type shallow: bool 0428 @param addrem: make each historical message an incremental difference 0429 from the first earlier historical message; see same-name parameter 0430 of L{msg_diff<diff.msg_diff>} for possible values 0431 @type addrem: string 0432 0433 @returns: the ascription history 0434 @rtype: [AscPoint*] 0435 """ 0436 0437 ahist = _collect_ascription_history_w(msg, acat, aconf, None, set(), 0438 shallow) 0439 0440 # If the message is not ascribed, 0441 # add it in front as modified by unknown user. 0442 if not ahist or not ascription_equal(msg, ahist[0].msg): 0443 a = AscPoint() 0444 a.type = AscPoint.ATYPE_MOD 0445 a.user = None 0446 a.msg = msg 0447 ahist.insert(0, a) 0448 0449 # Equip ascriptions with position markers, 0450 # to be able to see gaps possibly introduced by removals. 0451 pos = 1 0452 for a in ahist: 0453 a.pos = pos 0454 pos += 1 0455 0456 # Eliminate clean merges from history. 0457 if nomrg: 0458 ahist_r = [] 0459 for i in range(len(ahist) - 1): 0460 a, ao = ahist[i], ahist[i + 1] 0461 if ( a.type != AscPoint.ATYPE_MOD 0462 or not merge_modified(ao.msg, a.msg) 0463 ): 0464 ahist_r.append(a) 0465 ahist_r.append(ahist[-1]) 0466 ahist = ahist_r 0467 0468 # Eliminate contiguous chain of modifications equal under the filter, 0469 # except for the earliest in the chain. 0470 # (After elimination of clean merges.) 0471 if hfilter: 0472 def flt (msg): 0473 msg = MessageUnsafe(msg) 0474 msg.msgstr = list(map(hfilter, msg.msgstr)) 0475 return msg 0476 ahist_r = [] 0477 a_prevmod = None 0478 ahist.reverse() 0479 for a in ahist: 0480 if ( a.type != AscPoint.ATYPE_MOD or not a_prevmod 0481 or flt(a.msg).inv != a_prevmod.msg.inv 0482 ): 0483 ahist_r.append(a) 0484 if a.type == AscPoint.ATYPE_MOD: 0485 a_prevmod = AscPoint(a) 0486 a_prevmod.msg = flt(a.msg) 0487 ahist = ahist_r 0488 ahist.reverse() 0489 0490 # Reduce history to particular segments of diffs between modifications. 0491 # (After filtering). 0492 if addrem: 0493 a_nextmod = None 0494 for a in ahist: 0495 if a.type == AscPoint.ATYPE_MOD: 0496 if a_nextmod is not None: 0497 msg_ediff(a.msg, a_nextmod.msg, emsg=a_nextmod.msg, 0498 addrem=addrem) 0499 a_nextmod = a 0500 0501 return ahist 0502 0503 0504 def _collect_ascription_history_w (msg, acat, aconf, before, seenmsg, 0505 shallow=False): 0506 0507 ahist = [] 0508 0509 # Avoid circular paths. 0510 if msg.key in seenmsg: 0511 return ahist 0512 seenmsg.add(msg.key) 0513 0514 # Collect history from current ascription message. 0515 if msg in acat: 0516 amsg = acat[msg] 0517 for a in collect_ascription_history_segment(amsg, acat, aconf): 0518 if not before or a.date <= before.date: 0519 ahist.append(a) 0520 0521 if shallow: 0522 return ahist 0523 0524 # Continue into the past by pivoting around earliest message if fuzzy. 0525 amsg = ahist[-1].msg if ahist else msg 0526 if amsg.fuzzy and amsg.msgid_previous: 0527 pmsg = MessageUnsafe() 0528 for field in _id_fields: 0529 setattr(pmsg, field, amsg.get(field + "_previous")) 0530 # All ascriptions beyond the pivot must be older than the oldest so far. 0531 after = ahist and ahist[-1] or before 0532 ct_ahist = _collect_ascription_history_w(pmsg, acat, aconf, after, 0533 seenmsg) 0534 ahist.extend(ct_ahist) 0535 0536 return ahist 0537 0538 0539 def collect_ascription_history_segment (amsg, acat, aconf): 0540 """ 0541 Collect a segment of an ascription history. 0542 0543 C{amsg} is an ascription message from the ascription catalog C{acat}, 0544 falling under the ascription configuration C{aconf}, 0545 and it contains a part of the ascription history of some message. 0546 This function is used to get only that part of the ascription history. 0547 The ascription history segment is a list of L{AscPoint} objects, 0548 ordered from the newest to the oldest by date of ascription. 0549 0550 @param amsg: the ascription message from the ascription catalog 0551 @type amsg: L{Message_base<message.Message_base>} 0552 @param acat: the ascription catalog 0553 @type acat: L{Catalog<catalog.Catalog>} 0554 @param aconf: the ascription configuration which covers the catalogs 0555 @type aconf: L{AscConfig} 0556 0557 @returns: the ascription history segment 0558 @rtype: [AscPoint*] 0559 """ 0560 0561 ahist = [] 0562 spos = dict([(field, [0]) for field in _nonid_fields_tracked]) 0563 pvals = dict([(field, [[]]) for field in _nonid_fields_tracked]) 0564 for aflds in _parse_ascription_fields(amsg, acat, aconf): 0565 a = AscPoint() 0566 a.user, a.type, a.tag, a.date, a.slen, a.fuzz, a.obs = aflds 0567 if a.slen: # separator existing, reconstruct the fields 0568 shead = _field_separator_head(a.slen) 0569 pmsg = MessageUnsafe() 0570 for field in _id_fields: 0571 setattr(pmsg, field, amsg.get(field)) 0572 for field in _nonid_fields_tracked: 0573 amsg_seq = _get_as_sequence(amsg, field) 0574 pmsg_seq = [] 0575 for i in range(len(amsg_seq)): 0576 aval = amsg_seq[i] 0577 pval = _amsg_step_value(aval, shead, "\n", 0578 spos[field], pvals[field], i) 0579 # ...do not break if None, has to roll all spos items 0580 if pval is not None: 0581 while i >= len(pmsg_seq): 0582 pmsg_seq.append("") 0583 pmsg_seq[i] = pval 0584 _set_from_sequence(pmsg_seq, pmsg, field) 0585 else: 0586 pmsg = MessageUnsafe(ahist[-1].msg) # must exist 0587 if a.fuzz: 0588 pmsg.flag.add("fuzzy") 0589 elif "fuzzy" in pmsg.flag: 0590 pmsg.flag.remove("fuzzy") 0591 pmsg.obsolete = a.obs 0592 a.rmsg, a.msg = amsg, pmsg 0593 ahist.append(a) 0594 0595 # Sort history by date and put it in reverse. 0596 # If several ascriptions have same time stamps, preserve their order. 0597 ahist_ord = list(zip(ahist, list(range(len(ahist))))) 0598 ahist_ord.sort(key=lambda x: (x[0].date, x[1])) 0599 ahist_ord.reverse() 0600 ahist = [x[0] for x in ahist_ord] 0601 0602 return ahist 0603 0604 0605 def _parse_ascription_fields (amsg, acat, aconf): 0606 """ 0607 Get ascriptions from given ascription message as list of tuples 0608 C{(user, type, tag, date, seplen, isfuzzy, isobsolete)}, 0609 with date being a real C{datetime} object. 0610 """ 0611 0612 ascripts = [] 0613 for cmnt in amsg.auto_comment: 0614 p = cmnt.find(":") 0615 if p < 0: 0616 warning_on_msg(_("@info", 0617 "No type " 0618 "in ascription comment '%(cmnt)s'.", 0619 cmnt=cmnt), amsg, acat) 0620 continue 0621 atype = cmnt[:p].strip() 0622 atag = "" 0623 lst = atype.split(_atag_sep, 1) 0624 if len(lst) == 2: 0625 atype = lst[0].strip() 0626 atag = lst[1].strip() 0627 lst = cmnt[p+1:].split("|") 0628 if len(lst) < 2 or len(lst) > 3: 0629 warning_on_msg(_("@info", 0630 "Wrong number of descriptors " 0631 "in ascription comment '%(cmnt)s'.", 0632 cmnt=cmnt), amsg, acat) 0633 continue 0634 0635 auser = lst.pop(0).strip() 0636 if not auser: 0637 warning_on_msg(_("@info", 0638 "Malformed user string " 0639 "in ascription comment '%(cmnt)s'.", 0640 cmnt=cmnt), amsg, acat) 0641 continue 0642 if auser not in aconf.users: 0643 warning_on_msg(_("@info", 0644 "Unknown user " 0645 "in ascription comment '%(cmnt)s'.", 0646 cmnt=cmnt), amsg, acat) 0647 continue 0648 0649 datestr = lst.pop(0).strip() 0650 try: 0651 date = parse_datetime(datestr) 0652 except: 0653 warning_on_msg(_("@info", 0654 "Malformed date string " 0655 "in ascription comment '%(cmnt)s'.", 0656 cmnt=cmnt), amsg, acat) 0657 continue 0658 0659 # States are reset only on modification ascriptions, 0660 # in order to keep them for the following review ascriptions. 0661 if atype == AscPoint.ATYPE_MOD: 0662 isfuzz = False 0663 isobs = False 0664 seplen = 0 0665 if lst: 0666 tmp = lst.pop(0).strip() 0667 if _mark_fuzz in tmp: 0668 isfuzz = True 0669 tmp = tmp.replace(_mark_fuzz, "", 1) 0670 if _mark_obs in tmp: 0671 isobs = True 0672 tmp = tmp.replace(_mark_obs, "", 1) 0673 if tmp: 0674 try: 0675 seplen = int(tmp) 0676 except: 0677 warning_on_msg(_("@info", 0678 "Malformed separator length " 0679 "in ascription comment '%(cmnt)s'.", 0680 cmnt=cmnt), amsg, acat) 0681 continue 0682 0683 ascripts.append((auser, atype, atag, date, seplen, isfuzz, isobs)) 0684 0685 return ascripts 0686 0687 0688 def _amsg_step_value (aval, shead, stail, spos, pvals, i): 0689 0690 if i >= len(spos): 0691 spos.extend([0] * (i - len(spos) + 1)) 0692 if i >= len(pvals): 0693 pvals.extend([[] for x in range(i - len(pvals) + 1)]) 0694 p0 = spos[i] 0695 p1 = aval.find(shead, p0) 0696 p2 = aval.find(stail, p1 + 1) 0697 if p2 < 0: 0698 p2 = len(aval) 0699 spos[i] = p2 + len(stail) 0700 mods = aval[p1 + len(shead):p2] 0701 if _trsep_mod_eq in mods: 0702 q1 = mods.find(_trsep_mod_eq) + len(_trsep_mod_eq) 0703 q2 = q1 0704 while q2 < len(mods) and mods[q2].isdigit(): 0705 q2 += 1 0706 nrev = int(mods[q1:q2]) 0707 pval = pvals[i][nrev] 0708 else: 0709 if _trsep_mod_none in mods: 0710 pval = None 0711 else: 0712 pval = aval[p0:p1] 0713 pvals[i].append(pval) 0714 return pval 0715 0716 0717 _trsep_head = "|" 0718 _trsep_head_ext = "~" 0719 _trsep_mod_none = "x" 0720 _trsep_mod_eq = "e" 0721 0722 def _field_separator_head (length): 0723 0724 return _trsep_head + _trsep_head_ext * length 0725 0726 0727 def _needed_separator_length (msg): 0728 0729 goodsep = False 0730 seplen = 0 0731 while not goodsep: 0732 seplen += 1 0733 sephead = _field_separator_head(seplen) 0734 goodsep = True 0735 for field in _nonid_fields_tracked: 0736 values = msg.get(field) 0737 if values is None: 0738 continue 0739 if isinstance(values, str): 0740 values = [values] 0741 for value in values: 0742 if sephead in value: 0743 goodsep = False 0744 break 0745 if not goodsep: 0746 break 0747 0748 return seplen 0749 0750 0751 def _get_as_sequence (msg, field, asc=True): 0752 0753 if not asc and not msg.fuzzy and field in _fields_previous: 0754 # Ignore previous fields on non-ascription messages without fuzzy flag. 0755 return [] 0756 0757 msg_seq = msg.get(field) 0758 if msg_seq is None: 0759 msg_seq = [] 0760 elif field not in _multiple_fields: 0761 msg_seq = [msg_seq] 0762 elif field in _fields_comment: 0763 # Report comments as a single newline-delimited entry. 0764 if msg_seq: 0765 msg_seq = ["\n".join(msg_seq)] 0766 0767 return msg_seq 0768 0769 0770 def _set_from_sequence (msg_seq, msg, field): 0771 0772 if field not in _multiple_fields: 0773 # Single entry; set to given, or to None if no elements. 0774 msg_val = None 0775 if msg_seq: 0776 msg_val = msg_seq[0] 0777 multiple = False 0778 elif field in _fields_comment: 0779 # Comments treated as single newline-delimited entries; split. 0780 msg_val = [] 0781 if msg_seq: 0782 msg_val = msg_seq[0].split("\n") 0783 multiple = True 0784 else: 0785 # Straight sequence. 0786 msg_val = msg_seq 0787 multiple = True 0788 0789 if multiple and isinstance(msg, Message): 0790 msg_val = Monlist(msg_val) 0791 0792 setattr(msg, field, msg_val) 0793 0794 0795 # ----------------------------------------------------------------------------- 0796 # Writing ascriptions. 0797 0798 0799 def ascribe_modification (msg, user, dt, acat, aconf): 0800 """ 0801 Ascribe message modification. 0802 0803 @param msg: modified message which is being ascribed 0804 @type msg: L{Message_base<message.Message_base>} 0805 @param user: user to whom the ascription is made 0806 @type user: string 0807 @param dt: the time stamp when the ascription is made 0808 @type dt: datetime.datetime 0809 @param acat: the ascription catalogs 0810 @type acat: L{Catalog<catalog.Catalog>} 0811 @param aconf: the ascription configuration 0812 @type aconf: L{AscConfig} 0813 """ 0814 0815 _ascribe_any(msg, user, acat, AscPoint.ATYPE_MOD, [], aconf, dt) 0816 0817 0818 def ascribe_review (msg, user, dt, tags, acat, aconf): 0819 """ 0820 Ascribe message review. 0821 0822 @param msg: reviewed message which is being ascribed 0823 @type msg: L{Message_base<message.Message_base>} 0824 @param user: user to whom the ascription is made 0825 @type user: string 0826 @param dt: the time stamp when the ascription is made 0827 @type dt: datetime.datetime 0828 @param tags: review tags 0829 @type tags: [string*] 0830 @param acat: the ascription catalogs 0831 @type acat: L{Catalog<catalog.Catalog>} 0832 @param aconf: the ascription configuration 0833 @type aconf: L{AscConfig} 0834 """ 0835 0836 _ascribe_any(msg, user, acat, AscPoint.ATYPE_REV, tags, aconf, dt) 0837 0838 0839 _atag_sep = "/" 0840 _mark_fuzz = "f" 0841 _mark_obs = "o" 0842 0843 def _ascribe_any (msg, user, acat, atype, atags, aconf, dt=None): 0844 0845 # Create or retrieve ascription message. 0846 if msg not in acat: 0847 # Copy ID elements of the original message. 0848 amsg = Message() 0849 for field in _id_fields: 0850 setattr(amsg, field, getattr(msg, field)) 0851 # Append to the end of catalog. 0852 acat.add_last(amsg) 0853 else: 0854 # Retrieve existing ascription message. 0855 amsg = acat[msg] 0856 0857 # Reconstruct historical messages, from first to last. 0858 rahist = collect_ascription_history_segment(amsg, acat, aconf) 0859 rahist.reverse() 0860 0861 # Do any of non-ID elements differ to last historical message? 0862 if rahist: 0863 hasdiff_state = rahist[-1].msg.state() != msg.state() 0864 hasdiff_nonid = _has_nonid_diff(rahist[-1].msg, msg) 0865 else: 0866 hasdiff_nonid = True 0867 hasdiff_state = True 0868 hasdiff = hasdiff_nonid or hasdiff_state 0869 0870 # Add ascription comment. 0871 modstr = user + " | " + format_datetime(dt, wsec=True) 0872 modstr_wsep = modstr 0873 if hasdiff: 0874 wsep = "" 0875 if hasdiff_nonid: 0876 seplen = _needed_separator_length(msg) 0877 wsep += str(seplen) 0878 if msg.obsolete: 0879 wsep += _mark_obs 0880 if msg.fuzzy: 0881 wsep += _mark_fuzz 0882 if wsep: 0883 modstr_wsep += " | " + wsep 0884 first = True 0885 for atag in atags or [""]: 0886 field = atype 0887 if atag != "": 0888 field += _atag_sep + atag 0889 if first: 0890 _asc_append_field(amsg, field, modstr_wsep) 0891 first = False 0892 else: 0893 _asc_append_field(amsg, field, modstr) 0894 0895 # Add non-ID fields. 0896 if hasdiff_nonid: 0897 _add_nonid(amsg, msg, seplen, rahist) 0898 0899 # Update state. 0900 if msg.fuzzy: 0901 amsg.flag.add("fuzzy") 0902 else: 0903 amsg.flag.remove("fuzzy") 0904 if msg.obsolete: 0905 amsg.obsolete = True 0906 else: 0907 amsg.obsolete = False 0908 0909 0910 def _has_nonid_diff (pmsg, msg): 0911 0912 for field in _nonid_fields_tracked: 0913 msg_value = msg.get(field) 0914 if not msg.fuzzy and field in _fields_previous: 0915 # Ignore previous values in messages with no fuzzy flag. 0916 msg_value = None 0917 pmsg_value = pmsg.get(field) 0918 if msg_value != pmsg_value: 0919 return True 0920 0921 return False 0922 0923 0924 def _add_nonid (amsg, msg, slen, rahist): 0925 0926 shead = _field_separator_head(slen) 0927 nones = [_field_separator_head(x.slen) + _trsep_mod_none 0928 for x in rahist if x.slen] 0929 padnone = "\n".join(nones) 0930 0931 for field in _nonid_fields_tracked: 0932 0933 msg_seq = _get_as_sequence(msg, field, asc=False) 0934 amsg_seq = _get_as_sequence(amsg, field) 0935 0936 # Expand items to length in new message. 0937 for i in range(len(amsg_seq), len(msg_seq)): 0938 amsg_seq.append(padnone) 0939 0940 # Add to items. 0941 for i in range(len(amsg_seq)): 0942 if i < len(msg_seq): 0943 nmod = 0 0944 i_eq = None 0945 for a in rahist: 0946 if not a.slen: # no modification in this ascription 0947 continue 0948 if i_eq is None: 0949 msg_seq_p = _get_as_sequence(a.msg, field) 0950 if i < len(msg_seq_p) and msg_seq[i] == msg_seq_p[i]: 0951 i_eq = nmod 0952 # ...no break, need number of modifications. 0953 nmod += 1 0954 if i_eq is None: 0955 add = msg_seq[i] + shead 0956 else: 0957 add = shead + _trsep_mod_eq + str(i_eq) 0958 else: 0959 add = shead + _trsep_mod_none 0960 if amsg_seq[i]: 0961 amsg_seq[i] += "\n" 0962 amsg_seq[i] += add 0963 0964 _set_from_sequence(amsg_seq, amsg, field) 0965 0966 0967 fld_sep = ":" 0968 0969 def _asc_append_field (msg, field, value): 0970 0971 stext = "".join([field, fld_sep, " ", str(value)]) 0972 msg.auto_comment.append(stext) 0973 0974 0975 # ----------------------------------------------------------------------------- 0976 # Utilities for comparing and selecting ascriptions. 0977 0978 0979 def ascription_equal (msg1, msg2): 0980 """ 0981 Whether two messages are equal from the ascription viewpoint. 0982 0983 @param msg1: first message 0984 @type msg1: L{Message_base<message.Message_base>} 0985 @param msg2: second message 0986 @type msg2: L{Message_base<message.Message_base>} 0987 0988 @returns: C{True} if messages are equal, C{False} otherwise 0989 @rtype: bool 0990 """ 0991 0992 if msg1.state() != msg2.state(): 0993 return False 0994 if msg1.fuzzy: 0995 check_fields = _nonid_fields_eq_fuzzy 0996 else: 0997 check_fields = _nonid_fields_eq_nonfuzzy 0998 for field in check_fields: 0999 if msg1.get(field) != msg2.get(field): 1000 return False 1001 return True 1002 1003 1004 def merge_modified (msg1, msg2): 1005 """ 1006 Whether second message may have been derived from first 1007 by merging with templates. 1008 1009 @param msg1: first message 1010 @type msg1: L{Message_base<message.Message_base>} 1011 @param msg2: second message 1012 @type msg2: L{Message_base<message.Message_base>} 1013 1014 @returns: C{True} if C{msg2} is derived by merging from C{msg1}, 1015 C{False} otherwise 1016 @rtype: bool 1017 """ 1018 1019 # Anything can happen on merge when going from obsolete to current. 1020 if msg1.obsolete and not msg2.obsolete: 1021 return True 1022 1023 # Manual comments do not change on merge. 1024 if msg1.manual_comment != msg2.manual_comment: 1025 return False 1026 1027 # Current and previous original fields may have changed on merge, 1028 # depending on whether both messages are fuzzy, or only one, and which. 1029 if msg1.fuzzy == msg2.fuzzy: 1030 fields = msg1.fuzzy and _fields_previous or _fields_current 1031 for field in fields: 1032 if msg1.get(field) != msg2.get(field): 1033 return False 1034 else: 1035 fields = (msg1.fuzzy and list(zip(_fields_previous, _fields_current)) 1036 or list(zip(_fields_current, _fields_previous))) 1037 for field1, field2 in fields: 1038 if msg1.get(field1) != msg2.get(field2): 1039 return False 1040 1041 # Translation does not change on merge, except 1042 # on multiplication/reduction when plurality differs. 1043 if (msg1.msgid_plural is None) != (msg2.msgid_plural is None): 1044 if not msg1.fuzzy and not msg2.fuzzy: 1045 # Plurality cannot change between two non-fuzzy messages. 1046 return False 1047 if msg1.msgid_plural is not None: 1048 # Reduction to non-plural. 1049 if msg1.msgstr[0] != msg2.msgstr[0]: 1050 return False 1051 else: 1052 # Multiplication to plural. 1053 for msgstr in msg2.msgstr: 1054 if msgstr != msg1.msgstr[0]: 1055 return False 1056 else: 1057 if msg1.msgstr != msg2.msgstr: 1058 return False 1059 1060 return True 1061 1062 1063 def first_non_fuzzy (ahist, start=0): 1064 """ 1065 Find first non fuzzy message in the ascription history. 1066 1067 @param ahist: the ascription history 1068 @type ahist: [AscPoint*] 1069 @param start: position in history to start searching from 1070 @type start: int 1071 1072 @returns: index of first non-fuzzy message, or None if there is none such 1073 @rtype: int 1074 """ 1075 1076 for i in range(start, len(ahist)): 1077 hmsg = ahist[i].msg 1078 if hmsg and not hmsg.fuzzy: 1079 return i 1080 1081 return None 1082 1083 1084 def has_tracked_parts (msg): 1085 """ 1086 Check whether the message has any parts which are tracked for ascription. 1087 1088 For example, a pristine untranslated message is considered to have 1089 no tracked parts. 1090 1091 @returns: C{True} if there are any tracked parts, C{False} otherwise 1092 @rtype: bool 1093 """ 1094 1095 for part in _nonid_fields_tracked: 1096 pval = msg.get(part) 1097 if part not in _multiple_fields: 1098 if pval is not None and part != "msgid_plural": 1099 return True 1100 else: 1101 if part == "msgstr": 1102 for pval1 in pval: 1103 if pval1: 1104 return True 1105 elif pval: 1106 return True 1107 1108 return False 1109 1110 1111 # ----------------------------------------------------------------------------- 1112 # Argument parsing for selectors. 1113 1114 def parse_users (userspec, aconf): 1115 """ 1116 Parse ascription user specification. 1117 1118 The user specification is a comma-separated list of user names. 1119 If the list starts with tilde (~), all users defined in 1120 the ascription configuration but for those listed 1121 will be selected (inverted selection). 1122 1123 If an undefined user (according to ascription configuration) is mentioned, 1124 an exception is raised. 1125 1126 @param userspec: the user specification 1127 @type userspec: string 1128 @param aconf: the ascription configuration 1129 @type aconf: L{AscConfig} 1130 1131 @returns: selected user names 1132 @rtype: set(string*) 1133 """ 1134 1135 return _parse_fixed_set(userspec, aconf, aconf.users, 1136 t_("@info", 1137 "User '%(name)s' not defined in '%(file)s'.")) 1138 1139 1140 def parse_review_tags (tagspec, aconf): 1141 """ 1142 Parse review tag specification. 1143 1144 The tag specification is a comma-separated list of tags. 1145 If the list starts with tilde (~), all review tags defined in 1146 the ascription configuration but for those listed 1147 will be selected (inverted selection). 1148 1149 If an undefined tag (according to ascription configuration) is mentioned, 1150 an exception is raised. 1151 1152 @param tagspec: the review tag specification 1153 @type tagspec: string 1154 @param aconf: the ascription configuration 1155 @type aconf: L{AscConfig} 1156 1157 @returns: selected review tags 1158 @rtype: set(string*) 1159 """ 1160 1161 tags = _parse_fixed_set(tagspec, aconf, aconf.revtags, 1162 t_("@info", 1163 "Review tag '%(name)s' " 1164 "not defined in '%(file)s'.")) 1165 if not tags: 1166 tags = set([""]) 1167 1168 return tags 1169 1170 1171 def _parse_fixed_set (elstr, aconf, knownels, errfmt): 1172 1173 if not elstr: 1174 return set() 1175 1176 elstr = elstr.replace(" ", "") 1177 inverted = False 1178 if elstr.startswith("~"): 1179 inverted = True 1180 elstr = elstr[1:] 1181 1182 els = set(elstr.split(",")) 1183 for el in els: 1184 if el not in knownels: 1185 raise PologyError( 1186 errfmt.with_args(name=el, file=aconf.path).to_string()) 1187 if inverted: 1188 els = set(knownels).difference(els) 1189 1190 return els 1191 1192 # ----------------------------------------------------------------------------- 1193 # Caching for selectors. 1194 1195 _cache = {} 1196 1197 def cached_matcher (expr): 1198 """ 1199 Fetch a cached message matcher for the given expression, 1200 for use in ascription selectors. 1201 1202 When this function is called for the first time on a new expression, 1203 the matcher function is created and cached. 1204 On subsequent invocations with the same expression, 1205 the matcher is fetched from the cache rather than created anew. 1206 1207 @param expr: the matching expression; see 1208 L{make_msg_matcher<match.make_msg_matcher>} for details 1209 @type expr: string 1210 1211 @returns: the matcher function 1212 @rtype: (L{Message_base<message.Message_base>}, 1213 L{Catalog<catalog.Catalog>})->bool 1214 """ 1215 1216 key = ("matcher", expr) 1217 if key not in _cache: 1218 _cache[key] = make_msg_fmatcher(expr, abort=True) 1219 1220 return _cache[key] 1221 1222 1223 def cached_users (userspec, aconf, utype=None): 1224 """ 1225 Fetch a cached set of users for the given user specification, 1226 for use in ascription selectors. 1227 1228 When this function is called for the first time on a new combination 1229 of user specification C{userspec}, ascription configuration C{aconf}, 1230 and "user type" C{utype}, the specification is parsed and users collected. 1231 On subsequent invocations with the same combination, 1232 the user set is fetched from the cache rather than created anew. 1233 C{utype} is actually just an arbitrary string, 1234 for when you need to cache users by different categories. 1235 1236 @param userspec: the user specification; see L{parse_users} for details 1237 @type userspec: string 1238 @param aconf: the ascription configuration 1239 @type aconf: L{AscConfig} 1240 @param utype: user type 1241 @type utype: string 1242 1243 @returns: the set of users 1244 @rtype: set(string*) 1245 """ 1246 1247 key = ("users", userspec, aconf, utype) 1248 if key not in _cache: 1249 _cache[key] = parse_users(userspec, aconf) 1250 1251 return _cache[key] 1252 1253 1254 def cached_review_tags (tagspec, aconf): 1255 """ 1256 Fetch a cached set of review tags for the given tag specification, 1257 for use in ascription selectors. 1258 1259 When this function is called for the first time on a new combination 1260 of tag specification C{tagspec} and ascription configuration C{aconf}, 1261 the specification is parsed and tags collected. 1262 On subsequent invocations with the same combination, 1263 the tag set is fetched from the cache rather than created anew. 1264 1265 @param tagspec: the tag specification; see L{parse_review_tags} for details 1266 @type tagspec: string 1267 @param aconf: the ascription configuration 1268 @type aconf: L{AscConfig} 1269 1270 @returns: the set of tags 1271 @rtype: set(string*) 1272 """ 1273 1274 key = ("tags", tagspec, aconf) 1275 if key not in _cache: 1276 _cache[key] = parse_review_tags(tagspec, aconf) 1277 1278 return _cache[key] 1279 1280 1281 # ----------------------------------------------------------------------------- 1282 # Making selectors. 1283 1284 # Build compound selector out of list of specifications. 1285 # Selector specification is a string in format NAME:ARG1:ARG2:... 1286 # (instead of colon, separator can be any non-alphanumeric excluding 1287 # underscore and hyphen) 1288 def make_ascription_selector (selspecs, hist=False): 1289 """ 1290 Build compound ascription selector out of string specifications 1291 of basic selectors. 1292 1293 Selector specification string has the format NAME:ARG1:ARG2:... 1294 Instead of colon, separator can be any non-alphanumeric character 1295 used consistently, except for underscore and hyphen. 1296 The compound selector is obtained by constructing each 1297 basic selector according to the specification in turn, 1298 and linking them with AND-boolean semantics. 1299 1300 Parameter C{hist} determines whether the compound selector should 1301 be a shallow selector (C{True}) or a history selector (C{False}). 1302 If a history selector is required but cannot be made from 1303 the given composition of basic selectors, an exception is raised. 1304 1305 @param selspecs: specifications of basic selectors 1306 @type selspecs: [string*] 1307 @param hist: C{True} if the compound selector should be history selector, 1308 C{False} if it should be shallow selector 1309 @type hist: bool 1310 1311 @returns: the compound selector 1312 @rtype: (L{Message_base<message.Message_base>}, L{Catalog<catalog.Catalog>}, 1313 [AscPoint*], L{AscConfig})->bool (shallow), 1314 (...)->int/None (history) 1315 """ 1316 1317 # Component selectors. 1318 selectors = [] 1319 for selspec in selspecs: 1320 argsep = ":" 1321 for c in selspec: 1322 if not (c.isalpha() or c.isdigit() or c in ("_", "-")): 1323 argsep = c 1324 break 1325 lst = selspec.split(argsep) 1326 sname, sargs = lst[0], lst[1:] 1327 negated = False 1328 if sname.startswith("n"): 1329 sname = sname[1:] 1330 negated = True 1331 sfactory, can_hist = _selector_factories.get(sname, (None, False)) 1332 if not sfactory: 1333 raise PologyError( 1334 _("@info", 1335 "Unknown selector '%(sel)s'.", 1336 sel=sname)) 1337 if hist: 1338 if not can_hist: 1339 raise PologyError( 1340 _("@info", 1341 "Selector '%(sel)s' cannot be used " 1342 "as history selector.", 1343 sel=sname)) 1344 if negated: 1345 raise PologyError( 1346 _("@info", 1347 "Negated selectors (here '%(sel)s') cannot be used " 1348 "as history selectors.", 1349 sel=sname)) 1350 try: 1351 selector = sfactory(sargs) 1352 except PologyError as e: 1353 raise PologyError( 1354 _("@info", 1355 "Selector '%(sel)s' not created due to " 1356 "the following error:\n" 1357 "%(msg)s", 1358 sel=selspec, msg=str_to_unicode(str(e)))) 1359 if negated: 1360 selector = _negate_selector(selector) 1361 selectors.append((selector, selspec)) 1362 1363 # Compound selector. 1364 if hist: 1365 res0 = None 1366 else: 1367 res0 = False 1368 def cselector (msg, cat, ahist, aconf): 1369 res = res0 1370 for selector, selspec in selectors: 1371 try: 1372 res = selector(msg, cat, ahist, aconf) 1373 except PologyError as e: 1374 raise PologyError( 1375 _("@info", 1376 "Selector '%(sel)s' failed on message " 1377 "%(file)s:%(line)d:(#%(entry)d) " 1378 "with the following error:\n" 1379 "%(msg)s", 1380 sel=selspec, file=cat.filename, line=msg.refline, 1381 entry=msg.refentry, msg=str_to_unicode(str(e)))) 1382 if not res: 1383 return res 1384 return res 1385 1386 return cselector 1387 1388 1389 def _negate_selector (selector): 1390 1391 def negative_selector (*args): 1392 return not selector(*args) 1393 1394 return negative_selector 1395 1396 1397 _external_mods = {} 1398 1399 def import_ascription_extensions (modpath): 1400 """ 1401 Import extensions to ascription functionality from a Python module. 1402 1403 Additional selector factories can be introduced by defining 1404 the C{asc_selector_factories} dictionary, 1405 in which the key is the selector name, 1406 and the value a tuple of the selector factory function 1407 and the indicator of whether the selector can be used as 1408 a history selector or not. 1409 For example:: 1410 1411 asc_selector_factories = { 1412 # key: (function, can_be_used_as_history_selector), 1413 "specsel1": (selector_specsel1, True), 1414 "specsel2": (selector_specsel2, False), 1415 ... 1416 } 1417 1418 @param modpath: path to Python file 1419 @type modpath: string 1420 """ 1421 1422 # Load external module. 1423 try: 1424 with open(unicode_to_str(modpath)) as modfile: 1425 module_code = modfile.read() 1426 # ...unicode_to_str because of exec below. 1427 except IOError: 1428 raise PologyError( 1429 _("@info", 1430 "Cannot load external module '%(file)s'.", 1431 file=modpath)) 1432 # Load file into new module. 1433 modname = "mod" + str(len(_external_mods)) 1434 xmod = ModuleType(modname) 1435 exec(module_code, xmod.__dict__) 1436 modfile.close() 1437 _external_mods[modname] = xmod # to avoid garbage collection 1438 1439 # Collect everything collectable from the module. 1440 1441 xms = [] 1442 1443 xms.append("asc_selector_factories") 1444 selector_factories = getattr(xmod, xms[-1], None) 1445 if selector_factories is not None: 1446 _selector_factories.update(selector_factories) 1447 1448 # Warn of unknown externals. 1449 known_xms = set(xms) 1450 for xm in [x for x in dir(xmod) if x.startswith("asc_")]: 1451 if xm not in known_xms: 1452 warning(_("@info", 1453 "Unknown external resource '%(res)s' " 1454 "in module '%(file)s'.", 1455 res=xm, file=modpath)) 1456 1457 1458 # Registry of basic selector factories. 1459 _selector_factories = { 1460 # key: (function, can_be_used_as_history_selector), 1461 } 1462 1463 # ----------------------------------------------------------------------------- 1464 # Internal selector factories. 1465 # Use make_ascription_selector() to create selectors. 1466 1467 # NOTE: 1468 # Plain selectors should return True or False. 1469 # History selectors should return 1-based index into ascription history 1470 # when the appropriate historical message is found, and 0 otherwise. 1471 # In this way, when it is only necessary to test if a message is selected, 1472 # returns from both types of selectors can be tested for simple falsity/truth, 1473 # and non-zero integer return always indicates history selection. 1474 1475 def _selector_any (args): 1476 1477 if len(args) != 0: 1478 raise PologyError(_("@info", "Wrong number of arguments.")) 1479 1480 def selector (msg, cat, ahist, aconf): 1481 1482 return True 1483 1484 return selector 1485 1486 _selector_factories["any"] = (_selector_any, False) 1487 1488 1489 def _selector_active (args): 1490 1491 if len(args) != 0: 1492 raise PologyError(_("@info", "Wrong number of arguments.")) 1493 1494 def selector (msg, cat, ahist, aconf): 1495 1496 return msg.translated and not msg.obsolete 1497 1498 return selector 1499 1500 _selector_factories["active"] = (_selector_active, False) 1501 1502 1503 def _selector_current (args): 1504 1505 if len(args) != 0: 1506 raise PologyError(_("@info", "Wrong number of arguments.")) 1507 1508 def selector (msg, cat, ahist, aconf): 1509 1510 return not msg.obsolete 1511 1512 return selector 1513 1514 _selector_factories["current"] = (_selector_current, False) 1515 1516 1517 def _selector_branch (args): 1518 1519 if len(args) != 1: 1520 raise PologyError(_("@info", "Wrong number of arguments.")) 1521 branch = args[0] 1522 if not branch: 1523 raise PologyError(_("@info", "Branch ID must not be empty.")) 1524 branches = set(branch.split(",")) 1525 1526 def selector (msg, cat, ahist, aconf): 1527 1528 return bool(branches.intersection(parse_summit_branches(msg))) 1529 1530 return selector 1531 1532 _selector_factories["branch"] = (_selector_branch, False) 1533 1534 1535 def _selector_unasc (args): 1536 1537 if len(args) != 0: 1538 raise PologyError(_("@info", "Wrong number of arguments.")) 1539 1540 def selector (msg, cat, ahist, aconf): 1541 1542 # Do not consider pristine messages as unascribed. 1543 return ahist[0].user is None and has_tracked_parts(msg) 1544 1545 return selector 1546 1547 _selector_factories["unasc"] = (_selector_unasc, False) 1548 1549 1550 def _selector_fexpr (args): 1551 1552 if len(args) != 1: 1553 raise PologyError(_("@info", "Wrong number of arguments.")) 1554 expr = args[0] 1555 if not expr: 1556 raise PologyError(_("@info", "Match expression must not be empty.")) 1557 1558 def selector (msg, cat, ahist, aconf): 1559 1560 matcher = cached_matcher(expr) 1561 return bool(matcher(msg, cat)) 1562 1563 return selector 1564 1565 _selector_factories["fexpr"] = (_selector_fexpr, False) 1566 1567 1568 def _selector_e (args): 1569 1570 if len(args) != 1: 1571 raise PologyError(_("@info", "Wrong number of arguments.")) 1572 entry = args[0] 1573 if not entry or not entry.isdigit(): 1574 raise PologyError( 1575 _("@info", 1576 "Message entry number must be a positive integer.")) 1577 refentry = int(entry) 1578 1579 def selector (msg, cat, ahist, aconf): 1580 1581 return msg.refentry == refentry 1582 1583 return selector 1584 1585 _selector_factories["e"] = (_selector_e, False) 1586 1587 1588 def _selector_l (args): 1589 1590 if len(args) != 1: 1591 raise PologyError(_("@info", "Wrong number of arguments.")) 1592 line = args[0] 1593 if not line or not line.isdigit(): 1594 raise PologyError( 1595 _("@info", 1596 "Message line number must be a positive integer.")) 1597 refline = int(line) 1598 1599 def selector (msg, cat, ahist, aconf): 1600 1601 return abs(msg.refline - refline) <= 1 1602 1603 return selector 1604 1605 _selector_factories["l"] = (_selector_l, False) 1606 1607 1608 # Select messages between and including first and last reference by entry. 1609 # If first entry is not given, all messages to the last entry are selected. 1610 # If last entry is not given, all messages from the first entry are selected. 1611 def _selector_espan (args): 1612 1613 if not 1 <= len(args) <= 2: 1614 raise PologyError(_("@info", "Wrong number of arguments.")) 1615 first = args[0] 1616 last = args[1] if len(args) > 1 else "" 1617 if not first and not last: 1618 raise PologyError( 1619 _("@info", 1620 "At least one of the first and last message entry numbers " 1621 "must be given.")) 1622 if first and not first.isdigit(): 1623 raise PologyError( 1624 _("@info", 1625 "First message entry number must be a positive integer.")) 1626 if last and not last.isdigit(): 1627 raise PologyError( 1628 _("@info", 1629 "Last message entry number must be a positive integer.")) 1630 first_entry = (first and [int(first)] or [None])[0] 1631 last_entry = (last and [int(last)] or [None])[0] 1632 1633 def selector (msg, cat, ahist, aconf): 1634 1635 if first_entry is not None and msg.refentry < first_entry: 1636 return False 1637 if last_entry is not None and msg.refentry > last_entry: 1638 return False 1639 return True 1640 1641 return selector 1642 1643 _selector_factories["espan"] = (_selector_espan, False) 1644 1645 1646 # Select messages between and including first and last reference by line. 1647 # If first line is not given, all messages to the last line are selected. 1648 # If last line is not given, all messages from the first line are selected. 1649 def _selector_lspan (args): 1650 1651 if not 1 <= len(args) <= 2: 1652 raise PologyError(_("@info", "Wrong number of arguments.")) 1653 first = args[0] 1654 last = args[1] if len(args) > 1 else "" 1655 if not first and not last: 1656 raise PologyError( 1657 _("@info", 1658 "At least one of the first and last message line numbers " 1659 "must be given.")) 1660 if first and not first.isdigit(): 1661 raise PologyError( 1662 _("@info", 1663 "First message line number must be a positive integer.")) 1664 if last and not last.isdigit(): 1665 raise PologyError( 1666 _("@info", 1667 "Last message line number must be a positive integer.")) 1668 first_line = (first and [int(first)] or [None])[0] 1669 last_line = (last and [int(last)] or [None])[0] 1670 1671 def selector (msg, cat, ahist, aconf): 1672 1673 if first_line is not None and msg.refline < first_line: 1674 return False 1675 if last_line is not None and msg.refline > last_line: 1676 return False 1677 return True 1678 1679 return selector 1680 1681 _selector_factories["lspan"] = (_selector_lspan, False) 1682 1683 1684 def _selector_hexpr (args): 1685 1686 if not 1 <= len(args) <= 3: 1687 raise PologyError(_("@info", "Wrong number of arguments.")) 1688 expr = args[0] 1689 user_spec = args[1] if len(args) > 1 else "" 1690 addrem = args[2] if len(args) > 2 else "" 1691 if not expr: 1692 raise PologyError( 1693 _("@info", 1694 "Match expression cannot be empty.")) 1695 1696 def selector (msg, cat, ahist, aconf): 1697 1698 if ahist[0].user is None: 1699 return 0 1700 1701 matcher = cached_matcher(expr) 1702 users = cached_users(user_spec, aconf) 1703 1704 if not addrem: 1705 i = 0 1706 else: 1707 i = first_non_fuzzy(ahist, 0) 1708 if i is None: 1709 return 0 1710 1711 while i < len(ahist): 1712 a = ahist[i] 1713 if users and a.user not in users: 1714 i += 1 1715 continue 1716 1717 if not addrem: 1718 amsg = a.msg 1719 i_next = i + 1 1720 else: 1721 i_next = first_non_fuzzy(ahist, i + 1) 1722 if i_next is not None: 1723 amsg2 = ahist[i_next].msg 1724 else: 1725 amsg2 = MessageUnsafe(a.msg) 1726 for field in _nonid_fields_tracked: 1727 amsg2_value = amsg2.get(field) 1728 if amsg2_value is None: 1729 pass 1730 elif isinstance(amsg2_value, str): 1731 setattr(amsg2, field, None) 1732 else: 1733 amsg2_value = [""] * len(amsg2_value) 1734 i_next = len(ahist) 1735 amsg = MessageUnsafe(a.msg) 1736 msg_ediff(amsg2, amsg, emsg=amsg, addrem=addrem) 1737 1738 if matcher(amsg, cat): 1739 return i + 1 1740 1741 i = i_next 1742 1743 return 0 1744 1745 return selector 1746 1747 _selector_factories["hexpr"] = (_selector_hexpr, True) 1748 1749 1750 # Select last ascription (any, or by users). 1751 def _selector_asc (args): 1752 1753 if not 0 <= len(args) <= 1: 1754 raise PologyError(_("@info", "Wrong number of arguments.")) 1755 user_spec = args[0] if len(args) > 0 else "" 1756 1757 def selector (msg, cat, ahist, aconf): 1758 1759 if ahist[0].user is None: 1760 return 0 1761 1762 users = cached_users(user_spec, aconf) 1763 1764 hi_sel = 0 1765 for i in range(len(ahist)): 1766 a = ahist[i] 1767 if not users or a.user in users: 1768 hi_sel = i + 1 1769 break 1770 1771 return hi_sel 1772 1773 return selector 1774 1775 _selector_factories["asc"] = (_selector_asc, True) 1776 1777 1778 # Select last modification (any or by users). 1779 def _selector_mod (args): 1780 1781 if not 0 <= len(args) <= 1: 1782 raise PologyError(_("@info", "Wrong number of arguments.")) 1783 user_spec = args[0] if len(args) > 0 else "" 1784 1785 def selector (msg, cat, ahist, aconf): 1786 1787 if ahist[0].user is None: 1788 return 0 1789 1790 users = cached_users(user_spec, aconf) 1791 1792 hi_sel = 0 1793 for i in range(len(ahist)): 1794 a = ahist[i] 1795 if not a.user: 1796 continue 1797 if a.type == AscPoint.ATYPE_MOD and (not users or a.user in users): 1798 hi_sel = i + 1 1799 break 1800 1801 return hi_sel 1802 1803 return selector 1804 1805 _selector_factories["mod"] = (_selector_mod, True) 1806 1807 1808 # Select first modification (any or by m-users, and not by r-users) 1809 # after last review (any or by r-users, and not by m-users). 1810 def _selector_modar (args): 1811 1812 return _w_selector_modax(False, True, args, 3) 1813 1814 _selector_factories["modar"] = (_selector_modar, True) 1815 1816 1817 # Select first modification (any or by m-users, and not by mm-users) 1818 # after last modification (any or by mm-users, and not by m-users). 1819 def _selector_modam (args): 1820 1821 return _w_selector_modax(True, False, args, 2) 1822 1823 _selector_factories["modam"] = (_selector_modam, True) 1824 1825 1826 # Select first modification (any or by m-users, and not by rm-users) 1827 # after last review or modification (any or by m-users, and not by rm-users). 1828 def _selector_modarm (args): 1829 1830 return _w_selector_modax(True, True, args, 3) 1831 1832 _selector_factories["modarm"] = (_selector_modarm, True) 1833 1834 1835 # Select first modification of translation 1836 # (any or by m-users, and not by r-users) 1837 # after last review (any or by r-users, and not by m-users). 1838 def _selector_tmodar (args): 1839 1840 return _w_selector_modax(False, True, args, 3, True) 1841 1842 _selector_factories["tmodar"] = (_selector_tmodar, True) 1843 1844 1845 # Worker for builders of *moda* selectors. 1846 def _w_selector_modax (amod, arev, args, maxnarg, tronly=False): 1847 1848 if not 0 <= len(args) <= maxnarg: 1849 raise PologyError(_("@info", "Wrong number of arguments.")) 1850 muser_spec = args[0] if len(args) > 0 else "" 1851 rmuser_spec = args[1] if len(args) > 1 else "" 1852 atag_spec = args[2] if len(args) > 2 else "" 1853 1854 def selector (msg, cat, ahist, aconf): 1855 1856 if ahist[0].user is None: 1857 return 0 1858 1859 musers = cached_users(muser_spec, aconf, utype="m") 1860 rmusers = cached_users(rmuser_spec, aconf, utype="rm") 1861 atags = cached_review_tags(atag_spec, aconf) 1862 1863 hi_sel = 0 1864 for i in range(len(ahist)): 1865 a = ahist[i] 1866 1867 # Check if this message cancels further modifications. 1868 if ( ( (amod and a.type == AscPoint.ATYPE_MOD) 1869 or (arev and a.type == AscPoint.ATYPE_REV and a.tag in atags)) 1870 and (not rmusers or a.user in rmusers) 1871 and (not musers or a.user not in musers) 1872 ): 1873 break 1874 1875 # Check if this message is admissible modification. 1876 if ( a.type == AscPoint.ATYPE_MOD 1877 and (not musers or a.user in musers) 1878 and (not rmusers or a.user not in rmusers) 1879 ): 1880 # Cannot be a candidate if in translation-only mode and 1881 # there is no difference in translation to earlier message. 1882 ae = ahist[i + 1] if i + 1 < len(ahist) else None 1883 if not (tronly and ae and ae.msg.msgstr == a.msg.msgstr): 1884 hi_sel = i + 1 1885 1886 return hi_sel 1887 1888 return selector 1889 1890 1891 # Select last review (any or by users). 1892 def _selector_rev (args): 1893 1894 if not 0 <= len(args) <= 2: 1895 raise PologyError(_("@info", "Wrong number of arguments.")) 1896 user_spec = args[0] if len(args) > 0 else "" 1897 atag_spec = args[1] if len(args) > 1 else "" 1898 1899 def selector (msg, cat, ahist, aconf): 1900 1901 if ahist[0].user is None: 1902 return 0 1903 1904 users = cached_users(user_spec, aconf) 1905 atags = cached_review_tags(atag_spec, aconf) 1906 1907 hi_sel = 0 1908 for i in range(len(ahist)): 1909 a = ahist[i] 1910 if ( a.type == AscPoint.ATYPE_REV and a.tag in atags 1911 and (not users or a.user in users) 1912 ): 1913 hi_sel = i + 1 1914 break 1915 1916 return hi_sel 1917 1918 return selector 1919 1920 _selector_factories["rev"] = (_selector_rev, True) 1921 1922 1923 # Select first review (any or by r-users, and not by m-users) 1924 # before last modification (any or by m-users, and not by r-users). 1925 def _selector_revbm (args): 1926 1927 if not 0 <= len(args) <= 3: 1928 raise PologyError(_("@info", "Wrong number of arguments.")) 1929 ruser_spec = args[0] if len(args) > 0 else "" 1930 muser_spec = args[1] if len(args) > 1 else "" 1931 atag_spec = args[2] if len(args) > 2 else "" 1932 1933 def selector (msg, cat, ahist, aconf): 1934 1935 if ahist[0].user is None: 1936 return 0 1937 1938 rusers = cached_users(ruser_spec, aconf, utype="r") 1939 musers = cached_users(muser_spec, aconf, utype="m") 1940 atags = cached_review_tags(atag_spec, aconf) 1941 1942 hi_sel = 0 1943 can_select = False 1944 for i in range(len(ahist)): 1945 a = ahist[i] 1946 if ( a.type == AscPoint.ATYPE_MOD 1947 and (not musers or a.user in musers) 1948 and (not rusers or a.user not in rusers) 1949 ): 1950 # Modification found, enable selection of review. 1951 can_select = True 1952 if ( a.type == AscPoint.ATYPE_REV and a.tag in atags 1953 and (not rusers or a.user in rusers) 1954 and (not musers or a.user not in musers) 1955 ): 1956 # Review found, select it if enabled, and stop anyway. 1957 if can_select: 1958 hi_sel = i + 1 1959 break 1960 1961 return hi_sel 1962 1963 return selector 1964 1965 _selector_factories["revbm"] = (_selector_revbm, True) 1966 1967 1968 # Select first modification (any or by users) at or after given time. 1969 def _selector_modafter (args): 1970 1971 if not 0 <= len(args) <= 2: 1972 raise PologyError(_("@info", "Wrong number of arguments.")) 1973 time_spec = args[0] if len(args) > 0 else "" 1974 user_spec = args[1] if len(args) > 1 else "" 1975 if not time_spec: 1976 raise PologyError( 1977 _("@info", 1978 "Time specification cannot be empty.")) 1979 1980 date = parse_datetime(time_spec) 1981 1982 def selector (msg, cat, ahist, aconf): 1983 1984 if ahist[0].user is None: 1985 return 0 1986 1987 users = cached_users(user_spec, aconf) 1988 1989 hi_sel = 0 1990 for i in range(len(ahist) - 1, -1, -1): 1991 a = ahist[i] 1992 if ( a.type == AscPoint.ATYPE_MOD 1993 and (not users or a.user in users) 1994 and a.date >= date 1995 ): 1996 hi_sel = i + 1 1997 break 1998 1999 return hi_sel 2000 2001 return selector 2002 2003 _selector_factories["modafter"] = (_selector_modafter, True) 2004