File indexing completed on 2024-05-05 12:27:17
0001 #!/usr/bin/perl -w 0002 0003 # update script for kate-editor.org/syntax 0004 # SPDX-FileCopyrightText: 2020 Christoph Cullmann <cullmann@kde.org> 0005 # SPDX-License-Identifier: MIT 0006 0007 # needed things 0008 use FindBin; 0009 use Cwd; 0010 use File::Basename; 0011 use File::Copy; 0012 use File::Path qw(make_path remove_tree); 0013 use XML::Parser; 0014 0015 # be strict & warn 0016 use strict; 0017 use warnings; 0018 0019 # we need as parameter both the source and the build directory 0020 my $sourceDir = shift; 0021 my $buildDir = shift; 0022 if (!defined($sourceDir) || !defined($buildDir)) { 0023 die "Not all arguments provided, valid call: update-kate-editor-org.pl <source directory> <build directory>\n"; 0024 } 0025 0026 # output settings 0027 print "Using source directory: $sourceDir\n"; 0028 print "Using build directory: $buildDir\n"; 0029 0030 # switch to build directory, we do all our work there 0031 chdir($buildDir) || die "Failed to switch to build directory '$buildDir'!\n"; 0032 0033 # get kate-editor.org clone from invent.kde.org, update if already around 0034 if (-d "kate-editor-org") { 0035 print "Updating kate-editor.org clone...\n"; 0036 system("git", "-C", "kate-editor-org", "pull") == 0 || die "Failed to pull kate-editor-org.git!\n"; 0037 } else { 0038 print "Creating kate-editor.org clone...\n"; 0039 system("git", "clone", "git\@invent.kde.org:websites/kate-editor-org.git") == 0 || die "Failed to clone kate-editor-org.git!\n"; 0040 } 0041 0042 # 0043 # update of syntax definitions 0044 # beside the pure update site generation, we will create some web site with examples for all highlightings 0045 # 0046 0047 # try to get current frameworks version 0048 my $currentVersion; 0049 open (my $list, "<$sourceDir/CMakeLists.txt"); 0050 for (<$list>) { 0051 if ((my $version) = /^set\(KF_VERSION "[0-9]+\.([0-9]+)\.[0-9]+"\)/) { 0052 $currentVersion = $version; 0053 last; 0054 } 0055 } 0056 close $list; 0057 if (!defined($currentVersion)) { 0058 die "Failed to determine current version of syntax-highlighting framework!\n" 0059 } 0060 0061 # current maximal version 0062 print "Current version of syntax-highlighting: 5.$currentVersion\n"; 0063 0064 # purge old data in kate-editor.org clone 0065 my $staticSyntaxPath = "kate-editor-org/static/syntax"; 0066 my $staticSyntaxPathData = "$staticSyntaxPath/data/syntax"; 0067 remove_tree($staticSyntaxPath); 0068 if (-d $staticSyntaxPath) { 0069 die "Failed to delete '$staticSyntaxPath'!\n"; 0070 } 0071 make_path($staticSyntaxPathData); 0072 if (! -d $staticSyntaxPathData) { 0073 die "Failed to create '$staticSyntaxPathData'!\n"; 0074 } 0075 0076 # collect all known syntax files from the generated resource file and copy them over 0077 open my $resourceFile, "<$buildDir/data/syntax-data.qrc"; 0078 while (<$resourceFile>) { 0079 if ((my $file) = />(.*\.xml)<\/file>/) { 0080 copy($file, $staticSyntaxPathData) or die "Copy failed: $!"; 0081 } 0082 } 0083 close $resourceFile; 0084 0085 # copy over all html references as examples 0086 system("cp", "-rf", "$sourceDir/autotests/html", "$staticSyntaxPath/data/html") == 0 || die "Failed to copy HTML references!\n"; 0087 0088 # switch to kate-editor.org syntax directory now for post-processing 0089 chdir($staticSyntaxPath) || die "Failed to switch to '$staticSyntaxPath' directory!\n"; 0090 0091 # add new data to kate-editor.org git 0092 system("git", "add", "data") == 0 || die "Failed to add syntax files to git!\n"; 0093 0094 # setup XML parser with handler for start element 0095 my %languageAttributes = (); 0096 sub start_tag 0097 { 0098 # we only care for the language element, remember the attributes 0099 my($p, $tag, %attrs) = @_; 0100 if ($tag eq "language") { 0101 %languageAttributes = %attrs; 0102 $p->finish(); 0103 } 0104 } 0105 my $parser = XML::Parser->new( Handlers => { Start => \&start_tag }); 0106 0107 # read all syntax files and remember for their version infos! 0108 print "Parsing XML syntax/*.xml files...\n"; 0109 my %metaInfo; 0110 my %nameToFile; 0111 my $count = 0; 0112 foreach my $xmlFile (<data/syntax/*.xml>) { 0113 # parse the file 0114 %languageAttributes = (); 0115 $parser->parsefile( $xmlFile ); 0116 0117 # we need a name! 0118 my $name = $languageAttributes{'name'}; 0119 if (!defined($name)) { 0120 print "Skipping $xmlFile as name attribute is missing.\n"; 0121 next; 0122 } 0123 0124 # if we have no versions set, we can't handle this file! 0125 my $version = $languageAttributes{'version'}; 0126 if (!defined($version)) { 0127 print "Skipping $xmlFile as version attribute is missing.\n"; 0128 next; 0129 } 0130 my $kateversion = $languageAttributes{'kateversion'}; 0131 if (!defined($kateversion)) { 0132 print "Skipping $xmlFile as kateversion attribute is missing.\n"; 0133 next; 0134 } 0135 0136 # remember attributes 0137 # print "Remembering $xmlFile '$name' with version=$version & kateversion=$kateversion\n"; 0138 foreach my $key (keys %languageAttributes) { 0139 $metaInfo{$xmlFile}{$key} = $languageAttributes{$key}; 0140 } 0141 0142 # remember section => name => file mapping 0143 $nameToFile{$languageAttributes{'section'}}{$name} = $xmlFile; 0144 ++$count; 0145 } 0146 0147 # now: generate all needed update-*.xml files 0148 print "Generating XML update-*.xml files...\n"; 0149 my $minorVersion = 0; 0150 while ($minorVersion <= $currentVersion) { 0151 # generate one update file 0152 my $cVersion = "5.$minorVersion"; 0153 #print "Generation update-$cVersion.xml...\n"; 0154 open (my $update, ">update-$cVersion.xml"); 0155 print $update "<!DOCTYPE DEFINITIONS>\n"; 0156 print $update "<DEFINITIONS>\n"; 0157 foreach my $def (sort keys %metaInfo) { 0158 # is this definition allowed here? 0159 $_ = $metaInfo{$def}{kateversion}; 0160 if ((my $version) = /[0-9]+\.([0-9]+)/) { 0161 next if ($version > $minorVersion); 0162 } else { 0163 next; 0164 } 0165 print $update "<Definition name=\"$metaInfo{$def}{name}\" url=\"https://kate-editor.org/syntax/$def\" version=\"$metaInfo{$def}{version}\"/>\n"; 0166 } 0167 print $update "</DEFINITIONS>\n"; 0168 close $update; 0169 0170 # add to git 0171 system("git add update-$cVersion.xml") == 0 || die "Failed to add update-$cVersion.xml to git!\n"; 0172 0173 # next one 0174 ++$minorVersion; 0175 } 0176 0177 # parse the html files to match them to the highlighting they belong to 0178 # we just search for the dark variants and derive the names for the non-dark from that 0179 print "Parsing HTML example syntax/data/html/*.html files...\n"; 0180 my %nameToHTML; 0181 foreach my $htmlFile (<data/html/*.dark.html>, <data/html/.*.dark.html>) { 0182 my $name; 0183 open my $F, "<$htmlFile"; 0184 while (<$F>) { 0185 if (($name) = /name="generator" content="KF5::SyntaxHighlighting - Definition \((.*)\) - Theme/) { 0186 last; 0187 } 0188 } 0189 close $F; 0190 if (defined($name)) { 0191 $htmlFile =~ s/\.dark\.html//; 0192 if (defined($nameToHTML{$name})) { 0193 die "Duplicated test output found for '$name' (".$nameToHTML{$name}." vs. ".$htmlFile.").\ntesthighlighter_test in the framework should not have allowed that!\n"; 0194 } 0195 $nameToHTML{$name} = $htmlFile; 0196 } else { 0197 print "Skipping $htmlFile as proper generator meta information tag missing.\n"; 0198 } 0199 } 0200 0201 # we create some overview page, too, as markdown to brag about the amount of stuff we know .P 0202 print "Generating syntax.md overview page...\n"; 0203 my $syntax_md = "../../content/syntax.md"; 0204 open (my $syntax_page, ">$syntax_md"); 0205 print $syntax_page 0206 "---\n". 0207 "title: Syntax Highlighting\n". 0208 "hideMeta: true\n". 0209 "author: Christoph Cullmann\n". 0210 "date: 2019-08-24T12:12:12+00:00\n". 0211 "menu:\n". 0212 " main:\n". 0213 " weight: 110\n". 0214 " parent: menu\n". 0215 "---\n\n". 0216 "<!-- This page is auto-generated by \"make update_kate_editor_org\" in syntax-highlighting.git -->\n\n". 0217 "<p>Kate's highlighting is powered by the <a href=\"https://api.kde.org/frameworks/syntax-highlighting/html/\">KSyntaxHighlighting</a> framework and supports $count different languages.\n\n</p>". 0218 "<p>Below you can find a list of all supported languages together with example renderings of the highlighting both as light and dark theme HTML pages.\n\n</p>". 0219 "<p>Submissions of new language definitions & examples are very welcome, head over to the <a href=\"https://invent.kde.org/frameworks/syntax-highlighting#introduction\">KSyntaxHighlighting repository README</a> for details.\n\n</p>". 0220 "<table><tr><th>Category</th><th>Language</th><th colspan=2 nowrap>Light & Dark Theme Examples</th></tr>\n"; 0221 foreach my $section (sort keys %nameToFile) { 0222 foreach my $name (sort keys %{$nameToFile{$section}}) { 0223 my $file = $nameToFile{$section}{$name}; 0224 print $syntax_page "<tr><td>$section</td><td><a target=_blank href=\"/syntax/$file\">$name</a></td>\n"; 0225 0226 # link example output if existing 0227 if (defined($nameToHTML{$name})) { 0228 print $syntax_page "<td><a target=_blank href=\"/syntax/".$nameToHTML{$name}.".html\">Light Theme</a></td>"; 0229 print $syntax_page "<td><a target=_blank href=\"/syntax/".$nameToHTML{$name}.".dark.html\">Dark Theme</a></td>"; 0230 } else { 0231 print $syntax_page "<td colspan=2><center><a target=_blank href=\"https://invent.kde.org/frameworks/syntax-highlighting#adding-unit-tests-for-a-syntax-definition\">Submit Example</a></center></td>\n"; 0232 } 0233 print $syntax_page "</tr>\n"; 0234 } 0235 } 0236 print $syntax_page "</table>\n"; 0237 close($syntax_page); 0238 0239 # add to git 0240 system("git add $syntax_md") == 0 || die "Failed to add $syntax_md to git!\n"; 0241 0242 # 0243 # update of themes web site 0244 # this will generate an overview of all shipped themes with an example 0245 # 0246 0247 # switch back to build directory, we do all our work there 0248 chdir($buildDir) || die "Failed to switch to build directory '$buildDir'!\n"; 0249 0250 # purge old data in kate-editor.org clone 0251 my $staticThemePath = "kate-editor-org/static/themes"; 0252 remove_tree($staticThemePath); 0253 if (-d $staticThemePath) { 0254 die "Failed to delete '$staticThemePath'!\n"; 0255 } 0256 make_path($staticThemePath); 0257 if (! -d $staticThemePath) { 0258 die "Failed to create '$staticThemePath'!\n"; 0259 } 0260 0261 # copy over all html renderings as examples 0262 print "Updating theme example HTML files...\n"; 0263 system("cp", "-rf", "autotests/theme.html.output", "$staticThemePath/html") == 0 || die "Failed to copy autotests/theme.html.output references!\n"; 0264 0265 # switch over to git again 0266 chdir($staticThemePath) || die "Failed to switch to '$staticThemePath' directory!\n"; 0267 0268 # collect all themes with their test case 0269 print "Parsing theme kate-editor-org/static/themes/html/*.html files...\n"; 0270 my %themeToHTML; 0271 foreach my $htmlFile (<html/*.html>) { 0272 my $name; 0273 open my $F, "<$htmlFile"; 0274 while (<$F>) { 0275 if (($name) = /name="generator" content="KF5::SyntaxHighlighting - Definition \(.*\) - Theme \((.*)\)"/) { 0276 last; 0277 } 0278 } 0279 close $F; 0280 if (defined($name)) { 0281 $themeToHTML{$name} = $htmlFile; 0282 } else { 0283 print "Skipping $htmlFile as proper generator meta information tag missing.\n"; 0284 } 0285 } 0286 0287 # create HTML snippets one can embed into a page from the theme HTML pages 0288 # we will hash the stuff from lower case name to real name + file for better output order below 0289 my %themeToHTMLSnippet; 0290 foreach my $name (sort keys %themeToHTML) { 0291 # get full file 0292 open my $F, "<$themeToHTML{$name}"; 0293 my $fullFile = do { local $/; <$F> }; 0294 close $F; 0295 0296 # kill <body ...><pre> start and replace it with simple <pre> with body attribute 0297 $fullFile =~ s@.*<body style="(.*)"><pre>@<pre style="$1">@s; 0298 0299 # kill ending </pre>... and replace it with simple </pre> 0300 $fullFile =~ s@</pre></body></html>@</pre>@g; 0301 0302 # write snippet to disk 0303 my $snippetName = $themeToHTML{$name}; 0304 $snippetName =~ s/\.html/-snippet.html/; 0305 open my $OF, ">$snippetName"; 0306 print $OF $fullFile; 0307 close $OF; 0308 $themeToHTMLSnippet{lc($name)}{"name"} = $name; 0309 $themeToHTMLSnippet{lc($name)}{"file"} = $snippetName; 0310 } 0311 0312 # add html files 0313 system("git", "add", "html") == 0 || die "Failed to add theme HTML files to git!\n"; 0314 0315 # we create some overview page, too, as markdown to brag about the amount of stuff we know .P 0316 # we output sorted by lower case names as otherwise ayu and co. end up at the end... 0317 print "Generating themes.md overview page...\n"; 0318 my $themes_md = "../../content/themes.md"; 0319 open (my $themes_page, ">$themes_md"); 0320 print $themes_page 0321 "---\n". 0322 "title: Color Themes\n". 0323 "hideMeta: true\n". 0324 "author: Christoph Cullmann\n". 0325 "date: 2019-08-24T12:12:12+00:00\n". 0326 "menu:\n". 0327 " main:\n". 0328 " weight: 120\n". 0329 " parent: menu\n". 0330 "---\n\n". 0331 "<!-- This page is auto-generated by \"make update_kate_editor_org\" in syntax-highlighting.git -->\n\n". 0332 "<p>Kate's highlighting is powered by the <a href=\"https://api.kde.org/frameworks/syntax-highlighting/html/\">KSyntaxHighlighting</a> framework that contains ".scalar(keys %themeToHTML)." bundled color themes.\n". 0333 "Below you can find a list of all bundled themes together with an example rendering.\n\n</p>"; 0334 foreach my $lcName (sort keys %themeToHTMLSnippet) { 0335 my $name = $themeToHTMLSnippet{$lcName}{"name"}; 0336 my $file = $themeToHTMLSnippet{$lcName}{"file"}; 0337 print $themes_page "<h3>$name</h3>\n"; 0338 print $themes_page "{{% include \"/static/themes/$file\" %}}\n"; 0339 } 0340 close($themes_page); 0341 0342 # add to git 0343 system("git add $themes_md") == 0 || die "Failed to add $themes_md to git!\n";