From b3a7a6b36c695abc5515e806464b497997f59115 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 13 May 2016 11:36:41 +0100 Subject: [PATCH 1/6] Add a makefile Replace the python scripts for building the shared lib and tests with a Makefile, which makes it easier to handle a mix of C and C++. --- Makefile | 69 +++++++++++++++++++++++++++++++++++++++++ README.rst | 14 ++++++--- build_shared_library.py | 36 --------------------- test.py | 38 ----------------------- 4 files changed, 78 insertions(+), 79 deletions(-) create mode 100644 Makefile delete mode 100755 build_shared_library.py delete mode 100755 test.py diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a722b70 --- /dev/null +++ b/Makefile @@ -0,0 +1,69 @@ +#!/usr/bin/make -f + +BUILD_DIR := build +OPTIMIZE_FLAGS ?= -g -O3 +TEST_OPTIMIZE_FLAGS ?= -g -O0 +CC = gcc +TARGET := $(BUILD_DIR)/libolm.so + +SOURCES := $(wildcard src/*.cpp) $(wildcard src/*.c) +OBJECTS := $(patsubst src/%,$(BUILD_DIR)/%,$(patsubst %.c,%.o,$(patsubst %.cpp,%.o,$(SOURCES)))) +TEST_SOURCES := $(wildcard tests/test_*.cpp) $(wildcard tests/test_*.c) +TEST_BINARIES := $(patsubst tests/%,$(BUILD_DIR)/%,$(patsubst %.c,%,$(patsubst %.cpp,%,$(TEST_SOURCES)))) + +CPPFLAGS += -Iinclude -Ilib +CFLAGS += -Wall --std=c89 -fPIC +CXXFLAGS += -Wall --std=c++11 -fPIC +LDFLAGS += -Wall + +# generate .d files when compiling +CPPFLAGS += -MMD + +### per-target variables + +$(OBJECTS): CFLAGS += $(OPTIMIZE_FLAGS) +$(OBJECTS): CXXFLAGS += $(OPTIMIZE_FLAGS) +$(TARGET): LDFLAGS += $(OPTIMIZE_FLAGS) + +$(TEST_BINARIES): CPPFLAGS += -Itests/include +$(TEST_BINARIES): LDLIBS += -lolm +$(TEST_BINARIES): LDFLAGS += $(TEST_OPTIMIZE_FLAGS) -L$(BUILD_DIR) + +### top-level targets + +$(TARGET): $(OBJECTS) + $(CXX) $(LDFLAGS) --shared -fPIC $^ $(OUTPUT_OPTION) + +clean:; + rm -rf $(OBJECTS) $(OBJECTS:.o=.d) \ + $(TEST_BINARIES) $(TEST_BINARIES:=.d) \ + $(TARGET) + +build_tests: $(TEST_BINARIES) + +test: build_tests + for i in $(TEST_BINARIES); do \ + echo $$i; \ + LD_LIBRARY_PATH=$(BUILD_DIR) $$i || exit $$?; \ + done + +### rules for building objects +$(BUILD_DIR)/%.o: src/%.c + $(COMPILE.c) $(OUTPUT_OPTION) $< + +$(BUILD_DIR)/%.o: src/%.cpp + $(COMPILE.cc) $(OUTPUT_OPTION) $< + +$(BUILD_DIR)/%: tests/%.c + $(LINK.c) $< $(LOADLIBES) $(LDLIBS) -o $@ + +$(BUILD_DIR)/%: tests/%.cpp + $(LINK.cc) $< $(LOADLIBES) $(LDLIBS) -o $@ + + +### dependencies + +$(TEST_BINARIES): $(TARGET) + +-include $(OBJECTS:.o=.d) +-include $(TEST_BINARIES:=.d) diff --git a/README.rst b/README.rst index b2f48da..0075d84 100644 --- a/README.rst +++ b/README.rst @@ -14,14 +14,13 @@ To build olm as a shared library run: .. code:: bash - ./build_shared_library.py + make To run the tests run: .. code:: bash - ./test.py - + make test To build the javascript bindings, install emscripten from http://kripken.github.io/emscripten-site/ and then run: @@ -43,6 +42,11 @@ Design Olm is designed to be easy port to different platforms and to be easy to write bindings for. +It was originally implemented in C++, with a plain-C layer providing the public +API. As development has progressed, it has become clear that C++ gives little +advantage, and new functionality is being added in C, with C++ parts being +rewritten as the need ariases. + Error Handling ~~~~~~~~~~~~~~ @@ -78,13 +82,13 @@ the ratchet. While this decreases the performance it makes it much easier to compile the library for different architectures. What's an olm? -~~~~~~~~~~~~~~ +-------------- It's a really cool species of European troglodytic salamander. http://www.postojnska-jama.eu/en/come-and-visit-us/vivarium-proteus/ Legal Notice -~~~~~~~~~~~~ +------------ The software may be subject to the U.S. export control laws and regulations and by downloading the software the user certifies that he/she/it is diff --git a/build_shared_library.py b/build_shared_library.py deleted file mode 100755 index 911c7b3..0000000 --- a/build_shared_library.py +++ /dev/null @@ -1,36 +0,0 @@ -#! /usr/bin/env python -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import subprocess -import glob -import os -import sys - -if not os.path.exists("build"): - os.mkdir("build") - -source_files = glob.glob("src/*.cpp") - -compile_args = "g++ -Wall -O3 -Iinclude -Ilib --std=c++11 --shared -fPIC".split() -compile_args += source_files -compile_args += sys.argv[1:] - -library = "build/libolm.so" - -def run(args): - print " ".join(args) - subprocess.check_call(args) - -run(compile_args + ["-o", library]) diff --git a/test.py b/test.py deleted file mode 100755 index 83efe00..0000000 --- a/test.py +++ /dev/null @@ -1,38 +0,0 @@ -#! /usr/bin/env python -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import glob -import os -import subprocess -import sys - -if not os.path.exists("build/libolm.so"): - print >> sys.stderr, \ - "libolm has not been built. Run ./build_shared_library.py first." - exit(1) - -test_files = glob.glob("tests/test_*.cpp") - -compile_args = ("g++ -g -O0 -Itests/include -Iinclude -Ilib --std=c++11 "+ - "-L build").split() - -def run(args, *xargs, **kwargs): - print " ".join(args) - subprocess.check_call(args, *xargs, **kwargs) - -for test_file in test_files: - exe_file = "build/" + test_file[6:-4] - run(compile_args + [test_file, "-lolm", "-o", exe_file]) - run([exe_file], env={'LD_LIBRARY_PATH':'./build'}) From d4c8aae803a1fad46ded6e82a64575d9b6e40a7e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 20 May 2016 09:36:50 +0100 Subject: [PATCH 2/6] Add support for building the JS wrappers to the Makefile Now that we have C and C++, we need to split the compile and link steps (because we need different flags for the C and C++ files), so this is easier with a Makefile. --- Makefile | 45 +++++++++++++++++++++++++ README.rst | 2 +- javascript/build.py | 74 ----------------------------------------- javascript/package.json | 2 +- 4 files changed, 47 insertions(+), 76 deletions(-) delete mode 100755 javascript/build.py diff --git a/Makefile b/Makefile index a722b70..20a82a6 100644 --- a/Makefile +++ b/Makefile @@ -3,19 +3,38 @@ BUILD_DIR := build OPTIMIZE_FLAGS ?= -g -O3 TEST_OPTIMIZE_FLAGS ?= -g -O0 +JS_OPTIMIZE_FLAGS ?= -O3 CC = gcc +EMCC = emcc TARGET := $(BUILD_DIR)/libolm.so +JS_TARGET := javascript/olm.js + +JS_EXPORTED_FUNCTIONS := javascript/exported_functions.json + +PUBLIC_HEADERS := include/olm/olm.hh SOURCES := $(wildcard src/*.cpp) $(wildcard src/*.c) OBJECTS := $(patsubst src/%,$(BUILD_DIR)/%,$(patsubst %.c,%.o,$(patsubst %.cpp,%.o,$(SOURCES)))) TEST_SOURCES := $(wildcard tests/test_*.cpp) $(wildcard tests/test_*.c) TEST_BINARIES := $(patsubst tests/%,$(BUILD_DIR)/%,$(patsubst %.c,%,$(patsubst %.cpp,%,$(TEST_SOURCES)))) +JS_OBJECTS := $(patsubst %.o,%.js.bc,$(OBJECTS)) +JS_PRE := $(wildcard javascript/*pre.js) +JS_POST := $(wildcard javascript/*post.js) CPPFLAGS += -Iinclude -Ilib CFLAGS += -Wall --std=c89 -fPIC CXXFLAGS += -Wall --std=c++11 -fPIC LDFLAGS += -Wall +EMCCFLAGS = --closure 1 --memory-init-file 0 -s NO_FILESYSTEM=1 -s INVOKE_RUN=0 +# NO_BROWSER is kept for compatibility with emscripten 1.35.24, but is no +# longer needed. +EMCCFLAGS += -s NO_BROWSER=1 + +EMCC.c = $(EMCC) $(CFLAGS) $(CPPFLAGS) -c +EMCC.cc = $(EMCC) $(CXXFLAGS) $(CPPFLAGS) -c +EMCC_LINK = $(EMCC) $(LDFLAGS) $(EMCCFLAGS) + # generate .d files when compiling CPPFLAGS += -MMD @@ -29,14 +48,29 @@ $(TEST_BINARIES): CPPFLAGS += -Itests/include $(TEST_BINARIES): LDLIBS += -lolm $(TEST_BINARIES): LDFLAGS += $(TEST_OPTIMIZE_FLAGS) -L$(BUILD_DIR) +$(JS_OBJECTS): CFLAGS += $(JS_OPTIMIZE_FLAGS) +$(JS_OBJECTS): CXXFLAGS += $(JS_OPTIMIZE_FLAGS) +$(JS_TARGET): LDFLAGS += $(JS_OPTIMIZE_FLAGS) + ### top-level targets $(TARGET): $(OBJECTS) $(CXX) $(LDFLAGS) --shared -fPIC $^ $(OUTPUT_OPTION) +js: $(JS_TARGET) +.PHONY: js + +$(JS_TARGET): $(JS_OBJECTS) $(JS_PRE) $(JS_POST) $(JS_EXPORTED_FUNCTIONS) + $(EMCC_LINK) \ + --pre-js $(JS_PRE) --post-js $(JS_POST) \ + -s "EXPORTED_FUNCTIONS=@$(JS_EXPORTED_FUNCTIONS)" \ + $(JS_OBJECTS) -o $@ + clean:; rm -rf $(OBJECTS) $(OBJECTS:.o=.d) \ $(TEST_BINARIES) $(TEST_BINARIES:=.d) \ + $(JS_OBJECTS) $(JS_TARGET) \ + $(JS_EXPORTED_FUNCTIONS) \ $(TARGET) build_tests: $(TEST_BINARIES) @@ -47,6 +81,10 @@ test: build_tests LD_LIBRARY_PATH=$(BUILD_DIR) $$i || exit $$?; \ done +$(JS_EXPORTED_FUNCTIONS): $(PUBLIC_HEADERS) + perl -MJSON -ne '/(olm_[^( ]*)\(/ && push @f, "_$$1"; END { print encode_json \@f }' $^ > $@.tmp + mv $@.tmp $@ + ### rules for building objects $(BUILD_DIR)/%.o: src/%.c $(COMPILE.c) $(OUTPUT_OPTION) $< @@ -54,6 +92,12 @@ $(BUILD_DIR)/%.o: src/%.c $(BUILD_DIR)/%.o: src/%.cpp $(COMPILE.cc) $(OUTPUT_OPTION) $< +$(BUILD_DIR)/%.js.bc: src/%.c + $(EMCC.c) $(OUTPUT_OPTION) $< + +$(BUILD_DIR)/%.js.bc: src/%.cpp + $(EMCC.cc) $(OUTPUT_OPTION) $< + $(BUILD_DIR)/%: tests/%.c $(LINK.c) $< $(LOADLIBES) $(LDLIBS) -o $@ @@ -66,4 +110,5 @@ $(BUILD_DIR)/%: tests/%.cpp $(TEST_BINARIES): $(TARGET) -include $(OBJECTS:.o=.d) +-include $(JS_OBJECTS:.bc=.d) -include $(TEST_BINARIES:=.d) diff --git a/README.rst b/README.rst index 0075d84..0f3ae4f 100644 --- a/README.rst +++ b/README.rst @@ -26,7 +26,7 @@ To build the javascript bindings, install emscripten from http://kripken.github. .. code:: bash - javascript/build.py # builds javascript/olm.js + make js npm pack javascript # packages olm.js into olm-x.y.z.tgz Remember to make a tag after releasing a tarball: diff --git a/javascript/build.py b/javascript/build.py deleted file mode 100755 index f253acb..0000000 --- a/javascript/build.py +++ /dev/null @@ -1,74 +0,0 @@ -#! /usr/bin/env python -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import subprocess -import glob -import os -import sys -import re -import json - - -os.chdir(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) -source_files = glob.glob("src/*.cpp") -pre_js, = glob.glob("javascript/*pre.js") -post_js, = glob.glob("javascript/*post.js") - -functions = set() -RE_FUNCTION=re.compile("(olm_[^( ]*)\\(") -with open("include/olm/olm.hh") as header: - for line in header: - match = RE_FUNCTION.search(line) - if match: - functions.add(match.groups()[0]) - - -exported_functions = os.path.abspath("javascript/exported_functions.json") -with open(exported_functions, "w") as json_file: - json.dump(["_" + function for function in functions], json_file) - - -emcc = os.environ.get("EMCC", "emcc") -optimize_opts = os.environ.get("OPTIMIZE_FLAGS", "-O3") - -compile_args = [emcc] -compile_args += optimize_opts.split() -compile_args += ["-Wall"] -compile_args += """ - -Iinclude - -Ilib - -std=c++11 - --closure 1 - --memory-init-file 0 - -s NO_FILESYSTEM=1 - -s INVOKE_RUN=0 -""".split() -# NO_BROWSER is kept for compatibility with emscripten 1.35.24, but is no -# longer needed. -compile_args += ("-s","NO_BROWSER=1") -compile_args += source_files -compile_args += ("--pre-js", pre_js) -compile_args += ("--post-js", post_js) -compile_args += ("-s", "EXPORTED_FUNCTIONS=@" + exported_functions) -compile_args += sys.argv[1:] - -library = "javascript/olm.js" - -def run(args): - print args - print " ".join(args) - subprocess.check_call(args) - -run(compile_args + ["-o", library]) diff --git a/javascript/package.json b/javascript/package.json index dc16e45..710d46d 100644 --- a/javascript/package.json +++ b/javascript/package.json @@ -8,7 +8,7 @@ "README.md" ], "scripts": { - "build": "./build.py", + "build": "make -C .. js", "test": "echo \"Error: no test specified\" && exit 1" }, "repository": { From 488567a45f95cefd258f5f1a4cdf00c8848576d9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 20 May 2016 12:01:18 +0100 Subject: [PATCH 3/6] Use a version script to restrict symbols in the .so We want to stop aes_* and sha_* functions leaking out of our .so, as well as internal _olm_* symbols. This also means we need to link the unit tests against the objects. Possibly we should distinguish between unit tests and integration tests. --- Makefile | 17 ++++++++--------- version_script.ver | 9 +++++++++ 2 files changed, 17 insertions(+), 9 deletions(-) create mode 100644 version_script.ver diff --git a/Makefile b/Makefile index 20a82a6..9e01a1f 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,6 @@ $(OBJECTS): CXXFLAGS += $(OPTIMIZE_FLAGS) $(TARGET): LDFLAGS += $(OPTIMIZE_FLAGS) $(TEST_BINARIES): CPPFLAGS += -Itests/include -$(TEST_BINARIES): LDLIBS += -lolm $(TEST_BINARIES): LDFLAGS += $(TEST_OPTIMIZE_FLAGS) -L$(BUILD_DIR) $(JS_OBJECTS): CFLAGS += $(JS_OPTIMIZE_FLAGS) @@ -55,7 +54,9 @@ $(JS_TARGET): LDFLAGS += $(JS_OPTIMIZE_FLAGS) ### top-level targets $(TARGET): $(OBJECTS) - $(CXX) $(LDFLAGS) --shared -fPIC $^ $(OUTPUT_OPTION) + $(CXX) $(LDFLAGS) --shared -fPIC \ + -Wl,--version-script,version_script.ver \ + $(OUTPUT_OPTION) $(OBJECTS) js: $(JS_TARGET) .PHONY: js @@ -78,7 +79,7 @@ build_tests: $(TEST_BINARIES) test: build_tests for i in $(TEST_BINARIES); do \ echo $$i; \ - LD_LIBRARY_PATH=$(BUILD_DIR) $$i || exit $$?; \ + $$i || exit $$?; \ done $(JS_EXPORTED_FUNCTIONS): $(PUBLIC_HEADERS) @@ -98,17 +99,15 @@ $(BUILD_DIR)/%.js.bc: src/%.c $(BUILD_DIR)/%.js.bc: src/%.cpp $(EMCC.cc) $(OUTPUT_OPTION) $< -$(BUILD_DIR)/%: tests/%.c - $(LINK.c) $< $(LOADLIBES) $(LDLIBS) -o $@ +$(BUILD_DIR)/%: tests/%.c $(OBJECTS) + $(LINK.c) $< $(OBJECTS) $(LOADLIBES) $(LDLIBS) -o $@ -$(BUILD_DIR)/%: tests/%.cpp - $(LINK.cc) $< $(LOADLIBES) $(LDLIBS) -o $@ +$(BUILD_DIR)/%: tests/%.cpp $(OBJECTS) + $(LINK.cc) $< $(OBJECTS) $(LOADLIBES) $(LDLIBS) -o $@ ### dependencies -$(TEST_BINARIES): $(TARGET) - -include $(OBJECTS:.o=.d) -include $(JS_OBJECTS:.bc=.d) -include $(TEST_BINARIES:=.d) diff --git a/version_script.ver b/version_script.ver new file mode 100644 index 0000000..3aec5f6 --- /dev/null +++ b/version_script.ver @@ -0,0 +1,9 @@ +# this is a 'version script' for the linker which tells it to only export +# symbols starting 'olm_'. + +{ + global: + olm_*; + local: + *; +}; From 85f2d57e0859c40a702ef79c73357fb641fa63a2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 20 May 2016 15:28:54 +0100 Subject: [PATCH 4/6] Add an 'all' target to the Makefile So that we can build everything together. --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 9e01a1f..27abf45 100644 --- a/Makefile +++ b/Makefile @@ -53,6 +53,9 @@ $(JS_TARGET): LDFLAGS += $(JS_OPTIMIZE_FLAGS) ### top-level targets +lib: $(TARGET) +.PHONY: lib + $(TARGET): $(OBJECTS) $(CXX) $(LDFLAGS) --shared -fPIC \ -Wl,--version-script,version_script.ver \ @@ -86,6 +89,9 @@ $(JS_EXPORTED_FUNCTIONS): $(PUBLIC_HEADERS) perl -MJSON -ne '/(olm_[^( ]*)\(/ && push @f, "_$$1"; END { print encode_json \@f }' $^ > $@.tmp mv $@.tmp $@ +all: test js lib +.PHONY: lib + ### rules for building objects $(BUILD_DIR)/%.o: src/%.c $(COMPILE.c) $(OUTPUT_OPTION) $< From c8c5f35bb45db0547ea4c3e1d986051e47ba78d2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 16 May 2016 11:53:35 +0100 Subject: [PATCH 5/6] crypto.cpp: Fix comments These seem to be the wrong way around. --- src/crypto.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crypto.cpp b/src/crypto.cpp index da1d617..ffe2661 100644 --- a/src/crypto.cpp +++ b/src/crypto.cpp @@ -298,14 +298,14 @@ void olm::hkdf_sha256( salt = HKDF_DEFAULT_SALT; salt_length = sizeof(HKDF_DEFAULT_SALT); } - /* Expand */ + /* Extract */ hmac_sha256_key(salt, salt_length, hmac_key); hmac_sha256_init(&context, hmac_key); ::sha256_update(&context, input, input_length); hmac_sha256_final(&context, hmac_key, step_result); hmac_sha256_key(step_result, olm::SHA256_OUTPUT_LENGTH, hmac_key); - /* Extract */ + /* Expand */ hmac_sha256_init(&context, hmac_key); ::sha256_update(&context, info, info_length); ::sha256_update(&context, &iteration, 1); From ab7610087a0908640436d1703d33fc78715ca5e1 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 20 May 2016 16:50:14 +0100 Subject: [PATCH 6/6] Remove dep files for js objects on clean --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 27abf45..aa63c79 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,7 @@ $(JS_TARGET): $(JS_OBJECTS) $(JS_PRE) $(JS_POST) $(JS_EXPORTED_FUNCTIONS) clean:; rm -rf $(OBJECTS) $(OBJECTS:.o=.d) \ $(TEST_BINARIES) $(TEST_BINARIES:=.d) \ - $(JS_OBJECTS) $(JS_TARGET) \ + $(JS_OBJECTS) $(JS_OBJECTS:=.d) $(JS_TARGET) \ $(JS_EXPORTED_FUNCTIONS) \ $(TARGET)