Add graph references
This commit is contained in:
Abdelrahman Said
Executable
+567
@@ -0,0 +1,567 @@
|
||||
#! /usr/bin/env python3
|
||||
|
||||
# igraph library
|
||||
# Copyright (C) 2005-2021 The igraph development team
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
# 02110-1301 USA
|
||||
#
|
||||
###################################################################
|
||||
|
||||
"""DocBook XML generator for igraph.
|
||||
|
||||
The generator parses one or more input files for documentation chunks
|
||||
(embedded in the source code as Doxygen-style comments), and processes
|
||||
them with a set of regex-based rules. The processed chunks are then
|
||||
substituted into a template file containing <!-- doxrox-include -->
|
||||
directives.
|
||||
|
||||
When a template file is not provided, the generator will read the input
|
||||
files, process them with the ruleset and save a dictionary mapping chunk
|
||||
names to the corresponding processed chunks into a Python pickle. This
|
||||
can be used to speed up the processing of multiple input files as you can
|
||||
generate the chunks once and then re-use them for multiple input files.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from argparse import ArgumentParser
|
||||
from collections import defaultdict
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from fnmatch import fnmatch
|
||||
from hashlib import sha1
|
||||
from operator import itemgetter
|
||||
from pathlib import Path
|
||||
from pickle import dump, load
|
||||
from time import time
|
||||
from typing import Any, Callable, Dict, Iterator, List, Optional, Pattern
|
||||
|
||||
#: Constant indicating the start of a comment that doxrox.py will process
|
||||
DOXHEAD: str = r"/\*\*"
|
||||
|
||||
#: Stores whether we want verbose output
|
||||
verbose: bool = False
|
||||
|
||||
|
||||
def fatal(message: str, code: int = 1):
|
||||
"""Prints an error message and exits the program with the given error code."""
|
||||
print(message, file=sys.stderr)
|
||||
sys.exit(code)
|
||||
|
||||
|
||||
#########################################################################
|
||||
# The main function
|
||||
#########################################################################
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point of the script."""
|
||||
|
||||
global verbose
|
||||
|
||||
# get command line arguments
|
||||
parser = create_argument_parser()
|
||||
arguments = parser.parse_args()
|
||||
|
||||
outputfile: str = arguments.output_file
|
||||
inputs: List[str] = arguments.inputs
|
||||
|
||||
verbose = arguments.verbose
|
||||
|
||||
if (
|
||||
arguments.template_file in inputs
|
||||
or arguments.rules_file in inputs
|
||||
or outputfile in inputs
|
||||
):
|
||||
fatal("Special file is also used as an input file", 2)
|
||||
|
||||
# open the cache file if needed
|
||||
cache = ChunkCache(arguments.cache_file) if arguments.cache_file else None
|
||||
|
||||
# get all regular expressions
|
||||
rules: List[Rule]
|
||||
if arguments.rules_file:
|
||||
with operation("Reading regular expressions...") as op:
|
||||
rules = read_regex_rules_file(arguments.rules_file)
|
||||
op("{0} rules read".format(len(rules)))
|
||||
else:
|
||||
rules = []
|
||||
|
||||
# parse all input files and extract chunks, apply rules
|
||||
if arguments.chunk_file:
|
||||
with operation("Reading pickled chunks...") as op:
|
||||
try:
|
||||
with open(arguments.chunk_file, "rb") as f:
|
||||
all_chunks = load(f)
|
||||
except IOError:
|
||||
fatal("Error reading chunk file: " + arguments.chunk_file, 9)
|
||||
op("{0} chunks read".format(len(all_chunks)))
|
||||
else:
|
||||
all_chunks = {}
|
||||
|
||||
rule_timings = defaultdict(list)
|
||||
|
||||
for ifile in inputs:
|
||||
with operation("Parsing input file {0}...".format(ifile)) as op:
|
||||
try:
|
||||
with open(ifile, "r") as f:
|
||||
contents = f.read()
|
||||
except IOError:
|
||||
fatal("Error reading input file: " + ifile, 3)
|
||||
|
||||
if cache:
|
||||
key = cache.key_of(contents)
|
||||
chunks = cache.get(key)
|
||||
else:
|
||||
key, chunks = None, None
|
||||
|
||||
if chunks is not None:
|
||||
op("{0} chunks read from cache".format(len(chunks)))
|
||||
else:
|
||||
chunks = collect_chunks_from_input_file(
|
||||
ifile, contents, rules, rule_timings
|
||||
)
|
||||
op("{0} chunks parsed".format(len(chunks)))
|
||||
if key and cache:
|
||||
cache.put(key, chunks)
|
||||
|
||||
for name, chunk in chunks.items():
|
||||
if name in all_chunks:
|
||||
fatal(
|
||||
"Multiple files provide chunks for {0!r}".format(name), code=4
|
||||
)
|
||||
all_chunks[name] = chunk
|
||||
|
||||
if arguments.timing_stats and rule_timings:
|
||||
rule_timings = {name: sum(dts) / len(dts) for name, dts in rule_timings.items()}
|
||||
for name, dt in sorted(rule_timings.items(), key=itemgetter(1), reverse=True):
|
||||
print("{0}: {1:.3f}us".format(name, dt))
|
||||
print("======")
|
||||
|
||||
if cache:
|
||||
cache.close()
|
||||
|
||||
if arguments.template_file:
|
||||
# substitute the template file
|
||||
with operation("Reading template file..."):
|
||||
try:
|
||||
with open(arguments.template_file, "r") as tfile:
|
||||
tstring = tfile.read()
|
||||
except IOError:
|
||||
fatal("Error reading the template file: " + arguments.template_file, 7)
|
||||
|
||||
with operation("Substituting template file..."):
|
||||
chunk_iterator = re.finditer(
|
||||
r"<!--\s*doxrox-include\s+(\w+)\s*-->", tstring
|
||||
)
|
||||
outstring = []
|
||||
last = 0
|
||||
for match in chunk_iterator:
|
||||
try:
|
||||
chunk = all_chunks[match.group(1)]
|
||||
except KeyError:
|
||||
fatal("Chunk not found: {0}".format(match.group(1)), code=4)
|
||||
outstring.append(tstring[last : match.start()])
|
||||
outstring.append(chunk)
|
||||
last = match.end()
|
||||
outstring.append(tstring[last:])
|
||||
outstring = "".join(outstring)
|
||||
|
||||
# write output file
|
||||
with operation("Writing output file..."):
|
||||
try:
|
||||
with open(outputfile, "w") as ofile:
|
||||
ofile.write(outstring)
|
||||
except IOError:
|
||||
fatal("Error writing output file:" + outputfile, 8)
|
||||
else:
|
||||
# no template file given so just save the chunks as a pickle into the
|
||||
# output file
|
||||
with operation("Writing output file..."):
|
||||
try:
|
||||
with open(outputfile, "wb") as ofile:
|
||||
dump(all_chunks, ofile)
|
||||
except IOError:
|
||||
fatal("Error writing output file:" + outputfile, 5)
|
||||
|
||||
|
||||
#########################################################################
|
||||
# Argument parser
|
||||
#########################################################################
|
||||
|
||||
|
||||
def create_argument_parser() -> ArgumentParser:
|
||||
"""Creates the command line argument parser that the script uses."""
|
||||
parser = ArgumentParser(description=(sys.modules[__name__].__doc__ or "").strip())
|
||||
|
||||
parser.add_argument(
|
||||
"--cache",
|
||||
metavar="FILE",
|
||||
dest="cache_file",
|
||||
help="optional cache file to store chunks from already processed files",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-t",
|
||||
"--template",
|
||||
metavar="FILE",
|
||||
dest="template_file",
|
||||
help="template file to process",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-e",
|
||||
"--rules",
|
||||
metavar="FILE",
|
||||
dest="rules_file",
|
||||
help="file containing matching and replacement rules",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
metavar="FILE",
|
||||
dest="output_file",
|
||||
required=True,
|
||||
help="name of the output file",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v",
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
default=False,
|
||||
dest="verbose",
|
||||
help="enable verbose output",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--chunks",
|
||||
dest="chunk_file",
|
||||
metavar="FILE",
|
||||
help="name of a previously saved chunk file",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timing-stats",
|
||||
dest="timing_stats",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="print the average time it takes to process regex rules from the rules file",
|
||||
)
|
||||
parser.add_argument(
|
||||
"inputs", metavar="INPUT", nargs="*", help="input files to process"
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
#################
|
||||
# classes and functions to read the regular expression rules
|
||||
#################
|
||||
|
||||
|
||||
class RuleType(Enum):
|
||||
REPLACE = "replace"
|
||||
RUN = "run"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Rule:
|
||||
regex: Pattern[str]
|
||||
"""The regular expression that the rule will attempt to match."""
|
||||
|
||||
replacement: str
|
||||
"""The replacement string for the match, or the code to execute on the
|
||||
match.
|
||||
"""
|
||||
|
||||
type: RuleType
|
||||
"""Type of the rule"""
|
||||
|
||||
name: Optional[str]
|
||||
"""Name of the rule, for debugging purposes."""
|
||||
|
||||
glob: Optional[str] = None
|
||||
"""Optional glob pattern that specifies which input files the rule
|
||||
applies to.
|
||||
"""
|
||||
|
||||
def applies_to_filename(self, filename: str) -> bool:
|
||||
"""Returns whether the rule applies to files with the given name."""
|
||||
if self.glob:
|
||||
return fnmatch(filename, self.glob)
|
||||
else:
|
||||
return True
|
||||
|
||||
|
||||
def read_regex_rules_file(filename) -> List[Rule]:
|
||||
"""Parses the file containing the regex-based rules that we use to chop
|
||||
up the input source files into chunks that can later be fed into a
|
||||
DocBook document.
|
||||
|
||||
Parameters:
|
||||
filename: name of the input file
|
||||
|
||||
Returns:
|
||||
the rules that were parsed from the input file
|
||||
"""
|
||||
|
||||
rules: List[Rule] = []
|
||||
|
||||
def parse_error(lineno):
|
||||
"""Helper function to indicate a parse error at the given line."""
|
||||
fatal(
|
||||
"Parse error in regex file ({0}), line {1}".format(filename, lineno), code=4
|
||||
)
|
||||
|
||||
def store(
|
||||
rule: List[str],
|
||||
replacement: List[str],
|
||||
rule_name: Optional[str],
|
||||
rule_type: RuleType,
|
||||
glob: Optional[str],
|
||||
) -> None:
|
||||
"""Helper function to append the current rule to the result."""
|
||||
regex = re.compile("".join(rule), re.VERBOSE | re.MULTILINE | re.DOTALL)
|
||||
replacement_str = "".join(replacement)[:-1]
|
||||
rules.append(Rule(regex, replacement_str, rule_type, rule_name, glob))
|
||||
|
||||
mode = "empty"
|
||||
regex, replacement = [], []
|
||||
rule_name: Optional[str] = None
|
||||
rule_type: Optional[RuleType] = None
|
||||
glob: Optional[str] = None
|
||||
|
||||
try:
|
||||
with open(filename, "r") as f:
|
||||
for lineno, line in enumerate(f, 1):
|
||||
if line.startswith("REPLACE"):
|
||||
# a new pattern block starts
|
||||
if mode not in ("empty", "with"):
|
||||
parse_error(lineno)
|
||||
else:
|
||||
if regex and rule_type:
|
||||
store(regex, replacement, rule_name, rule_type, glob)
|
||||
regex.clear()
|
||||
replacement.clear()
|
||||
mode = "replace"
|
||||
|
||||
match = re.match(
|
||||
r"^REPLACE( IN (?P<glob>[^\s]+))?\s+-+\s+(?P<name>.*)\s+-",
|
||||
line,
|
||||
)
|
||||
rule_name = match.group("name") if match else None
|
||||
glob = match.group("glob") if match else None
|
||||
|
||||
elif line.startswith("WITH") or line.startswith("RUN"):
|
||||
# the second half of the pattern block starts
|
||||
if mode != "replace":
|
||||
parse_error(lineno)
|
||||
else:
|
||||
mode = "with"
|
||||
rule_type = (
|
||||
RuleType.REPLACE if line.startswith("WITH") else RuleType.RUN
|
||||
)
|
||||
|
||||
elif re.match(r"^\s*$", line):
|
||||
# empty line, do nothing
|
||||
pass
|
||||
|
||||
else:
|
||||
# normal line, append
|
||||
if mode == "replace":
|
||||
regex.append(line)
|
||||
elif mode == "with":
|
||||
replacement.append(line)
|
||||
else:
|
||||
parse_error(lineno)
|
||||
|
||||
if regex != "" and rule_type:
|
||||
store(regex, replacement, rule_name, rule_type, glob)
|
||||
|
||||
except IOError:
|
||||
fatal("Error reading regex file: " + filename, code=4)
|
||||
|
||||
return rules
|
||||
|
||||
|
||||
#################
|
||||
# parse an input file string
|
||||
#################
|
||||
def collect_chunks_from_input_file(
|
||||
path: str, strinput: str, rules: List[Rule], rule_timings
|
||||
) -> Dict[str, str]:
|
||||
result: Dict[str, str] = {}
|
||||
|
||||
# split the file
|
||||
chunks = re.split(DOXHEAD, strinput)
|
||||
chunks = chunks[1:]
|
||||
|
||||
# get the filename part of the path
|
||||
filename = os.path.basename(path)
|
||||
|
||||
# apply all rules to the chunks
|
||||
for chunk in chunks:
|
||||
name: Optional[str] = None
|
||||
|
||||
for rule in rules:
|
||||
start = time()
|
||||
|
||||
if not name and "name" in rule.regex.groupindex:
|
||||
# The regex might provide us with a chunk name so try figuring
|
||||
# out what the "name" group might match to
|
||||
matched = rule.regex.search(chunk)
|
||||
if matched:
|
||||
try:
|
||||
name = matched.group("name")
|
||||
except IndexError:
|
||||
name = ""
|
||||
|
||||
if rule.applies_to_filename(filename):
|
||||
if rule.type is RuleType.REPLACE:
|
||||
# This is a simple regex replacement rule
|
||||
try:
|
||||
chunk = rule.regex.sub(rule.replacement, chunk)
|
||||
except IndexError:
|
||||
print("Index error:" + chunk[0:60] + "...")
|
||||
print("Pattern:\n" + rule.regex.pattern)
|
||||
print("Current state:" + chunk[0:60] + "...")
|
||||
fatal("Parsing error", code=6)
|
||||
elif rule.type is RuleType.RUN:
|
||||
# This is a piece of Python code that has to be executed on
|
||||
# the part that matched
|
||||
matched = rule.regex.search(chunk)
|
||||
if matched:
|
||||
exec(rule.replacement)
|
||||
else:
|
||||
fatal("Invalid rule type: {0!r}".format(rule.type), code=6)
|
||||
|
||||
rule_timings[rule.name].append((time() - start) * 1000000)
|
||||
|
||||
if not name:
|
||||
# print("Chunk without a name ignored:" + ch[0:60] + "...")
|
||||
continue
|
||||
|
||||
result[name] = chunk.strip()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@contextmanager
|
||||
def operation(message: str) -> Iterator[Callable[[Any], None]]:
|
||||
"""Helper function to show progress messages for a potentially long-running
|
||||
operation in verbose mode.
|
||||
|
||||
Parameters:
|
||||
message (str): the message to show
|
||||
"""
|
||||
global verbose
|
||||
|
||||
if verbose:
|
||||
print(message, end="")
|
||||
|
||||
result = [None]
|
||||
|
||||
def set_result(obj: Any) -> None:
|
||||
result[0] = obj
|
||||
|
||||
success = False
|
||||
try:
|
||||
yield set_result
|
||||
success = True
|
||||
finally:
|
||||
if verbose and success:
|
||||
if result[0] is None:
|
||||
print(" done.")
|
||||
else:
|
||||
print(" done, {0}.".format(result[0]))
|
||||
|
||||
|
||||
class ChunkCache:
|
||||
"""Simple on-disk cache that stores SHA256 hashes of files along with the
|
||||
DocBook documentation chunks that were parsed from them.
|
||||
"""
|
||||
|
||||
_data: Optional[Dict[str, Dict[str, str]]]
|
||||
_dirty: bool
|
||||
_path: Path
|
||||
|
||||
def __init__(self, filename: str, hash=sha1):
|
||||
"""Constructor.
|
||||
|
||||
Parameters:
|
||||
filename: name of the file on the disk where the cache resides
|
||||
hash: the hash function to use
|
||||
"""
|
||||
self._data = None
|
||||
self._dirty = False
|
||||
self._hash = hash
|
||||
self._path = Path(filename)
|
||||
|
||||
def _load(self) -> None:
|
||||
"""Populates the in-memory copy of the cache from the disk."""
|
||||
if self._path.exists():
|
||||
try:
|
||||
with self._path.open("rb") as fp:
|
||||
self._data = load(fp)
|
||||
except (IOError, EOFError):
|
||||
# cache corrupted
|
||||
self._data = {}
|
||||
else:
|
||||
self._data = {}
|
||||
self._dirty = False
|
||||
|
||||
def close(self) -> None:
|
||||
"""Closes the cache and flushes its contents back to the disk if it
|
||||
changed recently.
|
||||
"""
|
||||
if self._dirty:
|
||||
self.flush()
|
||||
|
||||
def flush(self) -> None:
|
||||
"""Flushes the contents of the cache back to the disk."""
|
||||
with self._path.open("wb") as fp:
|
||||
dump(self._data, fp)
|
||||
self._dirty = False
|
||||
|
||||
def get(self, key: str) -> Optional[Dict[str, str]]:
|
||||
"""Returns the chunks associated to the file with the given key, or
|
||||
`None` if the key is not in the cache.
|
||||
"""
|
||||
if self._data is None:
|
||||
self._load()
|
||||
|
||||
assert self._data is not None
|
||||
return self._data.get(key)
|
||||
|
||||
def key_of(self, contents, encoding: str = "utf-8") -> str:
|
||||
"""Returns the hash key corresponding to the file with the given
|
||||
contents.
|
||||
"""
|
||||
if not isinstance(contents, bytes):
|
||||
contents = contents.encode(encoding)
|
||||
|
||||
key = self._hash()
|
||||
key.update(contents)
|
||||
return key.hexdigest()
|
||||
|
||||
def put(self, key: str, chunks: Dict[str, str]) -> None:
|
||||
"""Stores some chunks associated to the file with the given key."""
|
||||
assert self._data is not None
|
||||
self._data[key] = chunks
|
||||
self._dirty = True
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user