convert tests to python 3

makefile

@@ -132,39 +132,23 @@ else
 	cd $(PY_HOME)/DLLs && [ ! -f sqlite3-orig.dll ] && mv sqlite3.dll sqlite3-orig.dll || true
 	cp litetree-0.1.dll $(PY_HOME)/DLLs/sqlite3.dll
 	cp $(LMDBPATH)/lmdb.dll $(PY_HOME)/DLLs/lmdb.dll
-	cd test && python -mpip install lmdb
-	cd test && python test.py -v
-	cd test && python test-64bit-commit-ids.py -v
-endif
-else	# not Windows
-ifneq ($(shell python -c "import lmdb" 2> /dev/null; echo $$?),0)
-	sudo python -m easy_install cffi
-	cd test && sudo python -m easy_install lmdb
-ifneq ($(shell python -c "import lmdb" 2> /dev/null; echo $$?),0)
-	git clone --depth=1 https://github.com/dw/py-lmdb
-	cd py-lmdb && sudo LMDB_FORCE_CPYTHON=1 python setup.py install
-ifneq ($(shell python -c "import lmdb" 2> /dev/null; echo $$?),0)
-	sudo python -c "import cffi"
-	sudo python -c "import lmdb"
-endif
+	cd test && python3 -m pip install lmdb
+	cd test && python3 test.py -v
+	cd test && python3 test-64bit-commit-ids.py -v
 endif
+else
+ifneq ($(shell python3 -c "import lmdb" 2> /dev/null; echo $$?),0)
+	pip3 install lmdb
 endif
 ifeq ($(OS),OSX)
-ifneq ($(shell python -c "import pysqlite2.dbapi2" 2> /dev/null; echo $$?),0)
-ifneq ($(shell [ -d $(LIBPATH2) ]; echo $$?),0)
-	@echo "run 'sudo make install' first"
+ifneq ($(shell [ -d "$(LIBPATH2)" ]; echo $$?),0)
+	$(error On macOS, run sudo make install first. Tests need LiteTree under $(LIBPATH2). See README.md: "Compiling and installing" and "Running the Tests".)
 endif
-	git clone --depth=1 https://github.com/ghaering/pysqlite
-	cd pysqlite && echo "include_dirs=$(INCPATH)" >> setup.cfg
-	cd pysqlite && echo "library_dirs=$(LIBPATH2)" >> setup.cfg
-	cd pysqlite && python setup.py build
-	cd pysqlite && sudo python setup.py install
-endif
-	cd test && python test.py -v
-	cd test && python test-64bit-commit-ids.py -v
-else	# Linux
-	cd test && LD_LIBRARY_PATH=.. python test.py -v
-	cd test && LD_LIBRARY_PATH=.. python test-64bit-commit-ids.py -v
+	cd test && DYLD_LIBRARY_PATH="$(LIBPATH2):$(LMDBPATH)" python3 test.py -v
+	cd test && DYLD_LIBRARY_PATH="$(LIBPATH2):$(LMDBPATH)" python3 test-64bit-commit-ids.py -v
+else
+	cd test && LD_LIBRARY_PATH=..:/usr/local/lib python3 test.py -v
+	cd test && LD_LIBRARY_PATH=..:/usr/local/lib python3 test-64bit-commit-ids.py -v
 endif
 endif
 
@@ -176,22 +160,15 @@ else
 	cd $(PY_HOME)/DLLs && [ ! -f sqlite3-orig.dll ] && mv sqlite3.dll sqlite3-orig.dll || true
 	cp litetree-0.1.dll $(PY_HOME)/DLLs/sqlite3.dll
 	cp $(LMDBPATH)/lmdb.dll $(PY_HOME)/DLLs/lmdb.dll
-	cd test && python benchmark.py -v
+	cd test && python3 benchmark.py -v
 endif
 else ifeq ($(OS),OSX)
-ifneq ($(shell python -c "import pysqlite2.dbapi2" 2> /dev/null; echo $$?),0)
-ifneq ($(shell [ -d $(LIBPATH2) ]; echo $$?),0)
-	@echo "run 'sudo make install' first"
-endif
-	git clone --depth=1 https://github.com/ghaering/pysqlite
-	cd pysqlite && echo "include_dirs=$(INCPATH)" >> setup.cfg
-	cd pysqlite && echo "library_dirs=$(LIBPATH2)" >> setup.cfg
-	cd pysqlite && python setup.py build
-	cd pysqlite && sudo python setup.py install
+ifneq ($(shell [ -d "$(LIBPATH2)" ]; echo $$?),0)
+	$(error On macOS, run sudo make install first. Benchmark needs LiteTree under $(LIBPATH2). See README.md: "Compiling and installing".)
 endif
-	cd test && python benchmark.py -v
+	cd test && DYLD_LIBRARY_PATH="$(LIBPATH2):$(LMDBPATH)" python3 benchmark.py -v
 else
-	cd test && LD_LIBRARY_PATH=.. python benchmark.py -v
+	cd test && LD_LIBRARY_PATH=..:/usr/local/lib python3 benchmark.py -v
 endif
 
 # variables:

test/benchmark.py

@@ -3,18 +3,19 @@
 #
 import json
 import os
-import platform
-
-if platform.system() == "Darwin":
-    import pysqlite2.dbapi2 as sqlite3
-else:
-    import sqlite3
+import sys
+import sqlite3
 
 sqlite_version = "3.27.2"
 
 if sqlite3.sqlite_version != sqlite_version:
-    print "wrong SQLite version. expected: " + sqlite_version + " found: " + sqlite3.sqlite_version
-    quit()
+    print(
+        "wrong SQLite version. expected: "
+        + sqlite_version
+        + " found: "
+        + sqlite3.sqlite_version
+    )
+    sys.exit(1)
 
 def delete_file(filepath):
     if os.path.exists(filepath):

test/test-64bit-commit-ids.py

@@ -4,19 +4,14 @@
 import unittest
 import json
 import os
-import platform
 import lmdb
 import varint
-
-if platform.system() == "Darwin":
-    import pysqlite2.dbapi2 as sqlite3
-else:
-    import sqlite3
+import sqlite3
 
 sqlite_version = "3.27.2"
 
 if sqlite3.sqlite_version != sqlite_version:
-    print "wrong SQLite version. expected: " + sqlite_version + " found: " + sqlite3.sqlite_version
+    print("wrong SQLite version. expected: " + sqlite_version + " found: " + sqlite3.sqlite_version)
     import sys
     sys.exit(1)
 
@@ -26,6 +21,12 @@ def delete_file(filepath):
 
 v64bit_increment = 0xFFFFFFFE
 
+
+def _b(s):
+    """LMDB keys/values must be bytes under Python 3."""
+    return s if isinstance(s, bytes) else s.encode("ascii")
+
+
 class Test64bitCommitIds(unittest.TestCase):
 
     def test01_create_database(self):
@@ -91,35 +92,35 @@ def test02_replace_by_64bit_commit_ids(self):
 
         with env.begin(buffers=True) as txn:
 
-            value = txn.get('last_branch_id')
+            value = txn.get(_b("last_branch_id"))
             num_branches = varint.decode(value)[0]
             self.assertEqual(num_branches, 2)
 
             for branch_id in range(1, num_branches + 1):
-                pages_db.append(env.open_db('b' + str(branch_id) + '-pages'))
-                maxpg_db.append(env.open_db('b' + str(branch_id) + '-maxpage'))
+                pages_db.append(env.open_db(_b("b" + str(branch_id) + "-pages")))
+                maxpg_db.append(env.open_db(_b("b" + str(branch_id) + "-maxpage")))
                 self.assertEqual(len(pages_db) - 1, branch_id)
                 self.assertEqual(len(maxpg_db) - 1, branch_id)
 
         with env.begin(write=True, buffers=True) as txn:
 
-            value = txn.get('b1.name')
+            value = txn.get(_b("b1.name"))
             self.assertEqual(bytes(value).decode("utf-8"), "master\x00")
 
-            value = txn.get('b2.name')
+            value = txn.get(_b("b2.name"))
             self.assertEqual(bytes(value).decode("utf-8"), "test\x00")
 
             for branch_id in range(1, num_branches + 1):
-                prefix = 'b' + str(branch_id)
+                prefix = "b" + str(branch_id)
 
-                key = prefix + '.last_commit'
+                key = _b(prefix + ".last_commit")
                 value = txn.get(key)
                 last_commit = varint.decode(value)[0]
                 last_commit += v64bit_increment
                 value = varint.encode(last_commit)
                 txn.put(key, value)
 
-                key = prefix + '.source_commit'
+                key = _b(prefix + ".source_commit")
                 value = txn.get(key)
                 source_commit = varint.decode(value)[0]
                 if source_commit > 0:
@@ -133,7 +134,8 @@ def test02_replace_by_64bit_commit_ids(self):
                     res = varint.decode(key)
                     pgno = res[0]
                     size1 = res[1]
-                    res = varint.decode(key[size1:len(key)])
+                    keybuf = bytes(key)
+                    res = varint.decode(keybuf[size1 : len(keybuf)])
                     commit = res[0]
                     size2 = res[1]
                     if commit < v64bit_increment:

test/test.py

@@ -5,16 +5,12 @@
 import json
 import os
 import platform
-
-if platform.system() == "Darwin":
-    import pysqlite2.dbapi2 as sqlite3
-else:
-    import sqlite3
+import sqlite3
 
 sqlite_version = "3.27.2"
 
 if sqlite3.sqlite_version != sqlite_version:
-    print "wrong SQLite version. expected: " + sqlite_version + " found: " + sqlite3.sqlite_version
+    print("wrong SQLite version. expected: " + sqlite_version + " found: " + sqlite3.sqlite_version)
     import sys
     sys.exit(1)
 

test/varint.py

@@ -4,38 +4,51 @@
 # Copyright defined in LICENSE.txt
 #
 
+
+def _as_bytes(buf):
+    if isinstance(buf, memoryview):
+        return buf.tobytes()
+    if isinstance(buf, bytearray):
+        return bytes(buf)
+    if isinstance(buf, bytes):
+        return buf
+    # Python 2-style str (should not occur in Py3 tests)
+    return bytes(buf)
+
+
 def encode(num):
 
     if num < 0:
         raise ValueError("The number is negative")
 
     if num <= 240:
-        result = chr(num)
+        return bytes([num])
 
     elif num <= 2287:
         num -= 240
-        result = chr((num >> 8) + 241) + chr(num % 256)
+        return bytes([(num >> 8) + 241, num % 256])
 
     elif num <= 67823:
         num -= 2288
-        result = chr(249) + chr(num >> 8) + chr(num % 256)
+        return bytes([249, num >> 8, num % 256])
 
     else:
 
         if num > 0xFFFFFFFFFFFFFFFF:
             raise ValueError("The number is bigger than an unsigned 64-bit integer")
 
         # convert the 64-bit number to a buffer in big endian
-        buf = ''
+        buf = bytearray()
         shift = 56
         while shift >= 0:
-            buf += chr(num >> shift & 0xFF)
+            buf.append((num >> shift) & 0xFF)
             shift -= 8
+        buf = bytes(buf)
 
         # check how many zeros in the beginning
         start = 0
         for i in range(0, 8):
-            if ord(buf[i]) == 0:
+            if buf[i] == 0:
                 start += 1
             else:
                 break
@@ -44,41 +57,39 @@ def encode(num):
         num_bytes = 8 - start
 
         # build the result
-        result = chr(247 + num_bytes) + buf[start:8]
-
-
-    return result
+        return bytes([247 + num_bytes]) + buf[start:8]
 
 
 
 def decode(buf):
+    buf = _as_bytes(buf)
     size = len(buf)
     if size < 1: raise ValueError("Invalid varint")
-    first = ord(buf[0])
+    first = buf[0]
 
     if first <= 240:
         result = first
         num_bytes = 1
 
     elif first < 249:
         if size < 2: raise ValueError("Invalid varint")
-        second = ord(buf[1])
+        second = buf[1]
         result = 240 + ((first - 241) * 256) + second
         num_bytes = 2
 
     elif first == 249:
         if size < 3: raise ValueError("Invalid varint")
-        second = ord(buf[1])
-        third = ord(buf[2])
+        second = buf[1]
+        third = buf[2]
         result = 2288 + (second * 256) + third
         num_bytes = 3
 
     else:
         num_bytes = first - 247
         if size < num_bytes + 1: raise ValueError("Invalid varint")
-        result = ord(buf[1])
+        result = buf[1]
         for i in range(2, num_bytes + 1):
-            result = (result << 8) | ord(buf[i])
+            result = (result << 8) | buf[i]
         num_bytes += 1
 
     return (result, num_bytes)
@@ -88,11 +99,11 @@ def decode(buf):
 tests = 0
 
 def test_encode(num):
-    print 'testing ', num
-    buf = encode(num)
-    num2 = decode(buf)[0]
+    print('testing ', num)
+    b = encode(num)
+    num2 = decode(b)[0]
     if num2 != num:
-        print "FAILED!!!", num, num2
+        print("FAILED!!!", num, num2)
         quit()
     global tests
     tests += 1
@@ -102,5 +113,5 @@ def test_encode(num):
     while num < 0xFFFFFFFFFFFFFFFF:
         test_encode(num)
         num *= 3
-    print 'OK'
-    print tests, 'tests'
+    print('OK')
+    print(tests, 'tests')