From 6a35ad107c258cda50063c91334acd5783a672be Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 01:13:49 +0800 Subject: [PATCH 01/14] fix: in py3.7, as grey typo is not only a warning but error --- image_match/goldberg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/image_match/goldberg.py b/image_match/goldberg.py index da93ff8..e053b1c 100644 --- a/image_match/goldberg.py +++ b/image_match/goldberg.py @@ -236,14 +236,14 @@ def preprocess_image(image_or_path, bytestream=False, handle_mpo=False): return rgb2gray(np.asarray(img, dtype=np.uint8)) elif type(image_or_path) in string_types or \ type(image_or_path) is text_type: - return imread(image_or_path, as_grey=True) + return imread(image_or_path, as_gray=True) elif type(image_or_path) is bytes: try: img = Image.open(image_or_path) arr = np.array(img.convert('RGB')) except IOError: # try again due to PIL weirdness - return imread(image_or_path, as_grey=True) + return imread(image_or_path, as_gray=True) if handle_mpo: # take the first images from the MPO if arr.shape == (2,) and isinstance(arr[1].tolist(), MpoImageFile): From fec2b08fbf81599abbf30cad3a0626de9fca8c8c Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 01:21:55 +0800 Subject: [PATCH 02/14] feat: follow the latest mapping definition for es7.8 --- tests/test_elasticsearch_driver.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_elasticsearch_driver.py b/tests/test_elasticsearch_driver.py index 2a6daaa..63a12da 100644 --- a/tests/test_elasticsearch_driver.py +++ b/tests/test_elasticsearch_driver.py @@ -20,15 +20,16 @@ DOC_TYPE = 'image' MAPPINGS = { "mappings": { - DOC_TYPE: { - "dynamic": True, - "properties": { - "metadata": { - "type": "object", - "dynamic": True, + "properties": { + DOC_TYPE: { + "type": "nested", + "properties": { + "metadata": { + "type": "nested", "properties": { - "tenant_id": { "type": "keyword" } + "tenant_id": { "type": "keyword" } } + } } } } From 4814df89c2a2b67634d9e2a34ea4273416968b19 Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 15:32:05 +0800 Subject: [PATCH 03/14] feat: doc type is deprecated on 6.x --- image_match/elasticsearch_driver.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/image_match/elasticsearch_driver.py b/image_match/elasticsearch_driver.py index 1f105cc..a7f8857 100644 --- a/image_match/elasticsearch_driver.py +++ b/image_match/elasticsearch_driver.py @@ -53,24 +53,23 @@ def search_single_record(self, rec, pre_filter=None): rec.pop('metadata') # build the 'should' list - should = [{'term': {word: rec[word]}} for word in rec] + should = [{'term': {'{}.{}'.format(self.doc_type, word): rec[word]}} for word in rec] body = { 'query': { 'bool': {'should': should} }, - '_source': {'excludes': ['simple_word_*']} + '_source': {'excludes': ['{}.simple_word_*'.format(self.doc_type)]} } if pre_filter is not None: body['query']['bool']['filter'] = pre_filter res = self.es.search(index=self.index, - doc_type=self.doc_type, body=body, size=self.size, timeout=self.timeout)['hits']['hits'] - sigs = np.array([x['_source']['signature'] for x in res]) + sigs = np.array([x['_source'][self.doc_type]['signature'] for x in res]) if sigs.size == 0: return [] @@ -79,8 +78,8 @@ def search_single_record(self, rec, pre_filter=None): formatted_res = [{'id': x['_id'], 'score': x['_score'], - 'metadata': x['_source'].get('metadata'), - 'path': x['_source'].get('url', x['_source'].get('path'))} + 'metadata': x['_source'][self.doc_type].get('metadata'), + 'path': x['_source'][self.doc_type].get('url', x['_source'][self.doc_type].get('path'))} for x in res] for i, row in enumerate(formatted_res): @@ -91,7 +90,7 @@ def search_single_record(self, rec, pre_filter=None): def insert_single_record(self, rec, refresh_after=False): rec['timestamp'] = datetime.now() - self.es.index(index=self.index, doc_type=self.doc_type, body=rec, refresh=refresh_after) + self.es.index(index=self.index, body={ self.doc_type: rec }, refresh=refresh_after) def delete_duplicates(self, path): """Delete all but one entries in elasticsearch whose `path` value is equivalent to that of path. @@ -101,11 +100,11 @@ def delete_duplicates(self, path): matching_paths = [item['_id'] for item in self.es.search(body={'query': {'match': - {'path': path} + {'{}.path'.format(self.doc_type): path} } }, index=self.index)['hits']['hits'] - if item['_source']['path'] == path] + if item['_source'][self.doc_type]['path'] == path] if len(matching_paths) > 0: for id_tag in matching_paths[1:]: - self.es.delete(index=self.index, doc_type=self.doc_type, id=id_tag) + self.es.delete(index=self.index, id=id_tag) From 8164d6b683ba96329efca2594d9b38e52f336b82 Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 15:34:49 +0800 Subject: [PATCH 04/14] feat: migrate the query syntax to es 7.8 --- image_match/signature_database_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image_match/signature_database_base.py b/image_match/signature_database_base.py index 42eadd0..402791f 100644 --- a/image_match/signature_database_base.py +++ b/image_match/signature_database_base.py @@ -52,7 +52,7 @@ def search_single_record(self, rec, pre_filter=None): before applying the matching strategy For example: - { "term": { "metadata.category": "art" } } + { "term": { "image.metadata.category": "art" } } Returns: a formatted list of dicts representing matches. From 17ccf82978fb714f193a83764ff92d4ea288ba36 Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 15:37:11 +0800 Subject: [PATCH 05/14] feat: migrate the mapping syntax to latest version and make sure the type is keyword --- tests/test_elasticsearch_driver.py | 25 ++++++++++-------- ...elasticsearch_driver_metadata_as_nested.py | 26 ++++++++++--------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/tests/test_elasticsearch_driver.py b/tests/test_elasticsearch_driver.py index 63a12da..96aeee7 100644 --- a/tests/test_elasticsearch_driver.py +++ b/tests/test_elasticsearch_driver.py @@ -19,21 +19,24 @@ INDEX_NAME = 'test_environment_{}'.format(hashlib.md5(os.urandom(128)).hexdigest()[:12]) DOC_TYPE = 'image' MAPPINGS = { - "mappings": { - "properties": { - DOC_TYPE: { - "type": "nested", + "mappings": { "properties": { - "metadata": { - "type": "nested", - "properties": { - "tenant_id": { "type": "keyword" } + DOC_TYPE: { + "properties": { + "path": { + "type": "keyword" + }, + "metadata": { + "properties": { + "tenant_id": { + "type": "keyword", + } + } + } + } } - } } - } } - } } diff --git a/tests/test_elasticsearch_driver_metadata_as_nested.py b/tests/test_elasticsearch_driver_metadata_as_nested.py index 3c37c7c..a768f7b 100644 --- a/tests/test_elasticsearch_driver_metadata_as_nested.py +++ b/tests/test_elasticsearch_driver_metadata_as_nested.py @@ -19,21 +19,23 @@ INDEX_NAME = 'test_environment_{}'.format(hashlib.md5(os.urandom(128)).hexdigest()[:12]) DOC_TYPE = 'image' MAPPINGS = { - "mappings": { - DOC_TYPE: { - "dynamic": True, - "properties": { - "metadata": { - "type": "nested", - "dynamic": True, - "properties": { - "tenant_id": { "type": "keyword" }, - "project_id": { "type": "keyword" } + "mappings": { + "properties": { + DOC_TYPE: { + "properties": { + "path": { + "type": "keyword" + }, + "metadata": { + "properties": { + "tenant_id": { "type": "keyword" }, + "project_id": { "type": "keyword" } + } + } + } } } - } } - } } From a941990d2540ca43568b5d1558b26f0e999644e7 Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 15:39:33 +0800 Subject: [PATCH 06/14] fix: the error message has been changed in newer elasticsearch --- tests/test_elasticsearch_driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_elasticsearch_driver.py b/tests/test_elasticsearch_driver.py index 96aeee7..7338694 100644 --- a/tests/test_elasticsearch_driver.py +++ b/tests/test_elasticsearch_driver.py @@ -50,7 +50,7 @@ def setup_index(request, index_name): try: es.indices.create(index=index_name, body=MAPPINGS) except RequestError as e: - if e.error == u'index_already_exists_exception': + if e.error == u'resource_already_exists_exception': es.indices.delete(index_name) else: raise From ffb2c6273e65e1ef260e8f3e2815387eb80c6cea Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 15:40:18 +0800 Subject: [PATCH 07/14] test: update search filter syntax --- tests/test_elasticsearch_driver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_elasticsearch_driver.py b/tests/test_elasticsearch_driver.py index 7338694..2725296 100644 --- a/tests/test_elasticsearch_driver.py +++ b/tests/test_elasticsearch_driver.py @@ -193,15 +193,15 @@ def test_lookup_with_filter_by_metadata(ses): ) ses.add_image('test2.jpg', metadata=metadata2, refresh_after=True) - r = ses.search_image('test1.jpg', pre_filter={"term": {"metadata.tenant_id": "foo"}}) + r = ses.search_image('test1.jpg', pre_filter={"term": {'{}.metadata.tenant_id'.format(DOC_TYPE): "foo"}}) assert len(r) == 1 assert r[0]['metadata'] == metadata - r = ses.search_image('test1.jpg', pre_filter={"term": {"metadata.tenant_id": "bar-2"}}) + r = ses.search_image('test1.jpg', pre_filter={"term": {'{}.metadata.tenant_id'.format(DOC_TYPE): "bar-2"}}) assert len(r) == 1 assert r[0]['metadata'] == metadata2 - r = ses.search_image('test1.jpg', pre_filter={"term": {"metadata.tenant_id": "bar-3"}}) + r = ses.search_image('test1.jpg', pre_filter={"term": {'{}.metadata.tenant_id'.format(DOC_TYPE): "bar-3"}}) assert len(r) == 0 From b03c7b4c9de975cbf6ee4fee68615b51d8ab73fc Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 15:40:59 +0800 Subject: [PATCH 08/14] test: update search in multi filter query --- ...t_elasticsearch_driver_metadata_as_nested.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/tests/test_elasticsearch_driver_metadata_as_nested.py b/tests/test_elasticsearch_driver_metadata_as_nested.py index a768f7b..bbf4ede 100644 --- a/tests/test_elasticsearch_driver_metadata_as_nested.py +++ b/tests/test_elasticsearch_driver_metadata_as_nested.py @@ -124,16 +124,7 @@ def _metadata(tenant_id, project_id): ) def _nested_filter(tenant_id, project_id): - return { - "nested" : { - "path" : "metadata", - "query" : { - "bool" : { - "must" : [ - {"term": {"metadata.tenant_id": tenant_id}}, - {"term": {"metadata.project_id": project_id}} - ] - } - } - } - } \ No newline at end of file + return [ + {"term": {"image.metadata.tenant_id": tenant_id}}, + {"term": {"image.metadata.project_id": project_id}} + ] \ No newline at end of file From dfb126ab9b8a6258697d773bded4f5e979870447 Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 15:41:37 +0800 Subject: [PATCH 09/14] chore: upgrade depended elasticsearch version in setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e05c1f1..29c97db 100644 --- a/setup.py +++ b/setup.py @@ -89,7 +89,7 @@ def find_version(*file_paths): ], install_requires=[ 'scikit-image>=0.14', - 'elasticsearch>=5.0.0,<6.0.0', + 'elasticsearch>=7.0.0,<8.0.0', 'six>=1.11.0', ], tests_require=tests_require, From 4cd7fb794b6b4c1b38ee06cc7b48929699c9d085 Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 16:25:36 +0800 Subject: [PATCH 10/14] fix: import xml.etree.ElementTree since it doesn't auto import --- image_match/goldberg.py | 1 + 1 file changed, 1 insertion(+) diff --git a/image_match/goldberg.py b/image_match/goldberg.py index e053b1c..19674b1 100644 --- a/image_match/goldberg.py +++ b/image_match/goldberg.py @@ -10,6 +10,7 @@ from io import BytesIO import numpy as np import xml.etree +import xml.etree.ElementTree class CorruptImageError(RuntimeError): From 723c79c2080b6cff188aafc4b076b66f499b3d1a Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 16:30:14 +0800 Subject: [PATCH 11/14] chore: use py 3.5 for travis test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f145458..3ca2c94 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ services: language: python python: - - 3.4 + - 3.5 cache: pip From 0898b9b062a30acc55a7c9e98351c129bd0a573a Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 16:30:36 +0800 Subject: [PATCH 12/14] fix: use elasticsearch 7.8.0 for travis test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3ca2c94..af172b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ before_install: -e transport.host=127.0.0.1 -e xpack.security.enabled=false -e xpack.monitoring.enabled=false - docker.elastic.co/elasticsearch/elasticsearch:5.4.2 + docker.elastic.co/elasticsearch/elasticsearch:7.8.0 install: - pip install --upgrade pip codecov From 1422933decb79b709892d7abe8601e2b7df093ba Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 16:38:23 +0800 Subject: [PATCH 13/14] fix: pip install pytest for travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index af172b9..cac3611 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,7 @@ before_install: install: - pip install --upgrade pip codecov - pip install codecov + - pip install --upgrade pytest - pip install -e .[test] script: pytest --cov=image_match && codecov From e8cf7718500ebb536f7c1e850d2939c533c9ff22 Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Mon, 29 Jun 2020 00:43:26 +0800 Subject: [PATCH 14/14] chore: update difference result for test --- tests/test_goldberg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_goldberg.py b/tests/test_goldberg.py index 83857dc..7ba52d2 100644 --- a/tests/test_goldberg.py +++ b/tests/test_goldberg.py @@ -74,4 +74,4 @@ def test_difference(): sig1 = gis.generate_signature('test.jpg') sig2 = gis.generate_signature(test_diff_img_url) dist = gis.normalized_distance(sig1, sig2) - assert dist == 0.42263283502672722 + assert dist == 0.424549547059671