From d63fb70c693093d435d81755d576eb4c373613ac Mon Sep 17 00:00:00 2001
From: Deomid Ryabkov <rojer@cesanta.com>
Date: Tue, 27 Sep 2016 17:29:13 +0200
Subject: [PATCH] Publish the amalgamation tools
PUBLISHED_FROM=27ed0bd32e33252495b92361d2943a3450448f62
---
README.md | 4 ++
tools/README.md | 22 +++++++
tools/amalgam | 149 ++++++++++++++++++++++++++++++++++++++++++++++++
tools/unamalgam | 43 ++++++++++++++
4 files changed, 218 insertions(+)
create mode 100644 tools/README.md
create mode 100755 tools/amalgam
create mode 100755 tools/unamalgam
diff --git a/README.md b/README.md
index aee4b408d..ea1313d6d 100644
--- a/README.md
+++ b/README.md
@@ -73,4 +73,8 @@ To submit contributions, sign
[Cesanta CLA](https://docs.cesanta.com/contributors_la.shtml)
and send GitHub pull request. You retain the copyright on your contributions.
+## Working with the Source Code
+
+See [tools](https://github.com/cesanta/mongoose/tree/master/tools) directory.
+
[](https://github.com/cesanta/mongoose)
diff --git a/tools/README.md b/tools/README.md
new file mode 100644
index 000000000..e05db5c45
--- /dev/null
+++ b/tools/README.md
@@ -0,0 +1,22 @@
+# Amalgamation
+
+Mongoose is distributed as two files, `mongoose.c` and `mongoose.h` for ease of integration.
+However, when developing Mongoose itself, it can be quite difficult to work with them.
+Internally, these files are an _amalgamation_ of source an header modules.
+This directory contains utilities to split and re-constitute amalgamated files.
+
+Here's how `mongoose.c` can be split into its consituent parts:
+```
+$ tools/unamalgam mongoose.c
+=> mongoose/src/internal.h
+=> common/cs_dbg.h
+...
+```
+
+This produces directories and files under `mongoose/` and `common/` that are easeier to work with.
+It also produces `mongoose.c.manifest` which can later be used to reconstruct the file back:
+```
+$ tools/amalgam --prefix=MG --public-header=mongoose.h $(cat mongoose.c.manifest) > mongoose.c
+```
+
+The same applies to `mongoose.h`, except `--public-header` should be omitted during amalgamation.
diff --git a/tools/amalgam b/tools/amalgam
new file mode 100755
index 000000000..b60dc85af
--- /dev/null
+++ b/tools/amalgam
@@ -0,0 +1,149 @@
+#!/usr/bin/env python
+#
+# Generate a reversible amalgamation of several C source files
+# along with their required internal headers.
+#
+# This script assumes that there are a bunch of C files, a bunch
+# of private header files and one public header file.
+#
+# The script takes a list of C file names, parses `#include` directives
+# found in them and recursively resolves dependencies in such a way
+# that a header referenced from an included header will be emitted before the
+# header that depends on it. All headers will always be emitted before the
+# source files.
+#
+# The embedded include files usually contain private internals.
+# However sometimes it's necessary for some other tools or for advanced users
+# to have access to internal definitions. One such example is the generated
+# C source containing frozen heap. The amalgamation script will allow users
+# to include the amalgamated C file and cause extract only the internal headers:
+#
+# #define NS_EXPORT_INTERNAL_HEADERS
+# #include "v7.c"
+#
+# Where `NS` can be overridden via the --prefix flag.
+# This feature can be enabled with the --exportable-headers, and basically
+# all it does is to wrap the C body in a preprocessor guard.
+#
+# TODO(mkm): make it work also for mongoose where we also generate
+# the public header from a bunch of unamalgamated header files.
+# Currently this script can handle mongoose amalgamation because it doesn't
+# flip the --autoinc flag.
+#
+
+import argparse
+import re
+import sys
+import os
+from StringIO import StringIO
+
+parser = argparse.ArgumentParser(description='Produce an amalgamated source')
+parser.add_argument('--prefix', default="NS",
+ help='prefix for MODULE_LINES guard')
+parser.add_argument('--srcdir', default=".", help='source dir')
+parser.add_argument('--ignore', default="",
+ help='comma separated list of files to not amalgam')
+# hack, teach amalgam to render the LICENSE file instead
+parser.add_argument('--first', type=str, help='put this file in first position.'
+ ' Usually contains licensing info')
+parser.add_argument('--public-header', dest="public",
+ help='name of the public header file that will be'
+ ' included at the beginning of the file')
+parser.add_argument('--autoinc', action='store_true',
+ help='automatically embed include files')
+parser.add_argument('--exportable-headers', dest="export", action='store_true',
+ help='allow exporting internal headers')
+parser.add_argument('-I', default=".", dest='include_path', help='include path')
+parser.add_argument('sources', nargs='*', help='sources')
+
+class File(object):
+ def __init__(self, name):
+ self.name = name
+ self.buf = StringIO()
+ emit_file(self.buf, self.name)
+
+ def emit(self):
+ print self.buf.getvalue(),
+
+
+args = parser.parse_args()
+
+sources = []
+includes = []
+
+already_included = set()
+
+ignore_files = [i.strip() for i in args.ignore.split(',')]
+
+def should_ignore(name):
+ return (name in already_included
+ or not os.path.exists(resolve(name))
+ or name in ignore_files)
+
+def resolve(path):
+ if os.path.isabs(path) or os.path.exists(path):
+ p = path
+ else:
+ p = os.path.join(args.include_path, path)
+ if os.path.exists(p):
+ p = os.path.realpath(p).replace('%s/' % os.getcwd(), '')
+# print >>sys.stderr, '%s -> %s' % (path, p)
+ return p
+
+def emit_line_directive(out, name):
+ print >>out, '''#ifdef %(prefix)s_MODULE_LINES
+#line 1 "%(name)s"
+#endif''' % dict(
+ prefix = args.prefix,
+ name = resolve(name),
+)
+
+def emit_body(out, name):
+ path = resolve(name)
+ if not os.path.exists(path):
+ print >>out, '#include "%s"' % (name,)
+ return
+
+ with open(resolve(name)) as f:
+ for l in f:
+ match = re.match('(#include "(.*)")', l)
+ if match:
+ all, path = match.groups()
+ if args.autoinc:
+ if not should_ignore(path):
+ already_included.add(path)
+ includes.append(File(path))
+ print >>out, '/* Amalgamated: %s */' % (all,)
+ else:
+ print >>out, l,
+
+
+def emit_file(out, name):
+ emit_line_directive(out, name)
+ emit_body(out, name)
+
+for i in args.sources:
+ sources.append(File(i))
+
+if args.first:
+ for inc in reversed(args.first.split(',')):
+ for i, f in enumerate(includes):
+ if f.name == inc:
+ del includes[i]
+ includes.insert(0, f)
+ break
+
+# emitting
+
+if args.public:
+ print '#include "%s"' % (args.public)
+
+for i in includes:
+ i.emit()
+
+if args.export:
+ print '#ifndef %s_EXPORT_INTERNAL_HEADERS' % (args.prefix,)
+for i in sources:
+ i.emit()
+if args.export:
+ print '#endif /* %s_EXPORT_INTERNAL_HEADERS */' % (args.prefix,)
diff --git a/tools/unamalgam b/tools/unamalgam
new file mode 100755
index 000000000..2fd6a41eb
--- /dev/null
+++ b/tools/unamalgam
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+import sys
+import re
+import os
+
+cur_src = None
+in_mod = False
+ofile = None
+
+strip_re = re.compile(r'/\* Amalgamated: (.*) \*/')
+def clean(l):
+ return strip_re.sub(r'\1', l)
+
+manifest = []
+fname = sys.argv[1]
+with open(fname) as f:
+ for l in f:
+ if re.match('#ifdef .*_MODULE_LINES', l):
+ l = next(f)
+ g = re.match(r'#line [01] "(.*)"', l)
+ cur_src = g.group(1)
+
+ # if there is currently opened file, close it
+ if ofile:
+ ofile.close()
+
+ cur_src = re.sub(r'\.\./', '', cur_src)
+
+ # create directory for the next file if needed
+ cur_src_dir = os.path.dirname(cur_src)
+ if cur_src_dir != '' and not os.path.exists(cur_src_dir):
+ os.makedirs(cur_src_dir)
+
+ # open next file for writing
+ ofile = open(cur_src, "w")
+ print >>sys.stderr, '=> %s' % cur_src
+ manifest.append(cur_src)
+ next(f)
+ elif ofile:
+ ofile.write(clean(l))
+
+m = open('%s.manifest' % os.path.basename(fname), 'w')
+print >>m, '\n'.join(manifest)
--
GitLab