From afa9936eded5c8206ab81824aac4520fc69efaef Mon Sep 17 00:00:00 2001 From: SebastianGode Date: Tue, 13 Jun 2023 09:59:42 +0200 Subject: [PATCH 01/16] New file --- tools/index_metadata.py | 98 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 tools/index_metadata.py diff --git a/tools/index_metadata.py b/tools/index_metadata.py new file mode 100644 index 00000000..83287ce0 --- /dev/null +++ b/tools/index_metadata.py @@ -0,0 +1,98 @@ +import otc_metadata +import argparse +import logging +from opensearchpy import helpers as os_helpers +from opensearchpy import OpenSearch + + +def main(): + parser = argparse.ArgumentParser( + description="Create Index data for search inside OpenSearch" + ) + parser.add_argument( + "--target-environment", + required=True, + help="Environment to be used as a source", + ) + parser.add_argument( + '--delete-index', + action='store_true', + help='Option deletes old index with the same name and creates new ' + 'one.' + ) + parser.add_argument( + '--hosts', + metavar='', + nargs='+', + default=['localhost:9200'], + help='Provide one or multiple host:port values ' + 'separated by space for multiple hosts.\n' + 'Default: localhost:9200' + ) + parser.add_argument( + '--index', + metavar='', + default='test-index', + help="OpenSearch / ElasticSearch index name.\n" + 'Default: test-index' + ) + parser.add_argument( + '--username', + metavar='', + help='Username for the connection.' + ) + parser.add_argument( + '--password', + metavar='', + help='Password for the connection.' + ) + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + + data = getData( + environment=args.target_environment + ) + + indexedData = indexData( + deleteIndex=args.delete_index, + hosts=args.hosts, + index=args.index, + username=args.username, + password=args.password, + data=data + ) + +def getData(environment): + metadata = otc_metadata.Services() + return metadata.service_types_with_doc_types( + environment=environment + ) + +def indexData(deleteIndex, hosts, index, username, password, data): + hosts = generate_os_host_list(hosts) + client = OpenSearch( + hosts=hosts, + http_compress=True, + http_auth=(username, password), + use_ssl=True, + verify_certs=True, + ssl_assert_hostname=False, + ssl_show_warn=False + ) + + if deleteIndex == True: + client.indices.delete(index=index, ignore=[400, 404]) + +def generate_os_host_list(hosts): + host_list = [] + for host in hosts: + raw_host = host.split(':') + if len(raw_host) != 2: + raise Exception('--hosts parameter does not match the following ' + 'format: hostname:port') + json_host = {'host': raw_host[0], 'port': int(raw_host[1])} + host_list.append(json_host) + return host_list + +def create_index(client) \ No newline at end of file -- 2.34.1 From a2f7e401b619249b5449781cfaefeca1e35cac01 Mon Sep 17 00:00:00 2001 From: SebastianGode Date: Tue, 13 Jun 2023 10:10:36 +0200 Subject: [PATCH 02/16] Error fixes --- tools-requirements.txt | 1 + tools/index_metadata.py | 23 ++++++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/tools-requirements.txt b/tools-requirements.txt index 5fee53b0..1bddc556 100644 --- a/tools-requirements.txt +++ b/tools-requirements.txt @@ -4,3 +4,4 @@ requests jinja2 dirsync cookiecutter +opensearch-py diff --git a/tools/index_metadata.py b/tools/index_metadata.py index 83287ce0..218e76cc 100644 --- a/tools/index_metadata.py +++ b/tools/index_metadata.py @@ -1,11 +1,12 @@ import otc_metadata import argparse import logging -from opensearchpy import helpers as os_helpers from opensearchpy import OpenSearch -def main(): +metadata = otc_metadata.Services() + +def parse_args(): parser = argparse.ArgumentParser( description="Create Index data for search inside OpenSearch" ) @@ -39,16 +40,23 @@ def main(): parser.add_argument( '--username', metavar='', + required=True, help='Username for the connection.' ) parser.add_argument( '--password', metavar='', + required=True, help='Password for the connection.' ) args = parser.parse_args() + return args + +def main(): + logging.basicConfig(level=logging.DEBUG) + args = parse_args() data = getData( environment=args.target_environment @@ -64,7 +72,6 @@ def main(): ) def getData(environment): - metadata = otc_metadata.Services() return metadata.service_types_with_doc_types( environment=environment ) @@ -82,7 +89,9 @@ def indexData(deleteIndex, hosts, index, username, password, data): ) if deleteIndex == True: - client.indices.delete(index=index, ignore=[400, 404]) + delete_index(client, index) + + create_index(client, index, data) def generate_os_host_list(hosts): host_list = [] @@ -95,4 +104,8 @@ def generate_os_host_list(hosts): host_list.append(json_host) return host_list -def create_index(client) \ No newline at end of file +def create_index(client, index, data): + return client.indices.create(index, body=data) + +def delete_index(client, index): + return client.indices.delete(index=index, ignore=[400, 404]) \ No newline at end of file -- 2.34.1 From fd50baa0dc87f02339001d38af2498d018793b55 Mon Sep 17 00:00:00 2001 From: SebastianGode Date: Tue, 13 Jun 2023 10:16:22 +0200 Subject: [PATCH 03/16] Fix pep8 --- tools/index_metadata.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tools/index_metadata.py b/tools/index_metadata.py index 218e76cc..95bf6045 100644 --- a/tools/index_metadata.py +++ b/tools/index_metadata.py @@ -6,6 +6,7 @@ from opensearchpy import OpenSearch metadata = otc_metadata.Services() + def parse_args(): parser = argparse.ArgumentParser( description="Create Index data for search inside OpenSearch" @@ -53,8 +54,9 @@ def parse_args(): args = parser.parse_args() return args + def main(): - + logging.basicConfig(level=logging.DEBUG) args = parse_args() @@ -62,7 +64,7 @@ def main(): environment=args.target_environment ) - indexedData = indexData( + indexData( deleteIndex=args.delete_index, hosts=args.hosts, index=args.index, @@ -71,11 +73,13 @@ def main(): data=data ) + def getData(environment): return metadata.service_types_with_doc_types( environment=environment ) + def indexData(deleteIndex, hosts, index, username, password, data): hosts = generate_os_host_list(hosts) client = OpenSearch( @@ -88,11 +92,12 @@ def indexData(deleteIndex, hosts, index, username, password, data): ssl_show_warn=False ) - if deleteIndex == True: + if deleteIndex is True: delete_index(client, index) - + create_index(client, index, data) + def generate_os_host_list(hosts): host_list = [] for host in hosts: @@ -104,8 +109,10 @@ def generate_os_host_list(hosts): host_list.append(json_host) return host_list + def create_index(client, index, data): return client.indices.create(index, body=data) + def delete_index(client, index): - return client.indices.delete(index=index, ignore=[400, 404]) \ No newline at end of file + return client.indices.delete(index=index, ignore=[400, 404]) -- 2.34.1 From 90b841ddccdf084250710a680731f3451ea65e83 Mon Sep 17 00:00:00 2001 From: SebastianGode Date: Tue, 13 Jun 2023 10:30:08 +0200 Subject: [PATCH 04/16] Debug --- tools/index_metadata.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/index_metadata.py b/tools/index_metadata.py index 95bf6045..2be4ff59 100644 --- a/tools/index_metadata.py +++ b/tools/index_metadata.py @@ -60,10 +60,12 @@ def main(): logging.basicConfig(level=logging.DEBUG) args = parse_args() + logging.debug("Obtaining data from otc_metadata") data = getData( environment=args.target_environment ) + logging.debug("Indexing data into OpenSearch") indexData( deleteIndex=args.delete_index, hosts=args.hosts, @@ -93,9 +95,12 @@ def indexData(deleteIndex, hosts, index, username, password, data): ) if deleteIndex is True: + logging.debug("Deleting Index") delete_index(client, index) + logging.debug("Started creating Index") create_index(client, index, data) + logging.debug("Finished creating Index") def generate_os_host_list(hosts): @@ -116,3 +121,6 @@ def create_index(client, index, data): def delete_index(client, index): return client.indices.delete(index=index, ignore=[400, 404]) + + +main() \ No newline at end of file -- 2.34.1 From 2aa0175b6ef164af8cb8e4cdf8e2ee65817378e0 Mon Sep 17 00:00:00 2001 From: SebastianGode Date: Tue, 13 Jun 2023 10:34:36 +0200 Subject: [PATCH 05/16] Json --- tools/index_metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/index_metadata.py b/tools/index_metadata.py index 2be4ff59..5a1c2b58 100644 --- a/tools/index_metadata.py +++ b/tools/index_metadata.py @@ -2,6 +2,7 @@ import otc_metadata import argparse import logging from opensearchpy import OpenSearch +import json metadata = otc_metadata.Services() @@ -116,7 +117,7 @@ def generate_os_host_list(hosts): def create_index(client, index, data): - return client.indices.create(index, body=data) + return client.indices.create(index, body=json.dumps(data)) def delete_index(client, index): -- 2.34.1 From 7c866e7a4342f84e8dbe3c10a35773b73a949145 Mon Sep 17 00:00:00 2001 From: Sebastian Gode Date: Tue, 13 Jun 2023 08:42:03 +0000 Subject: [PATCH 06/16] Removed json --- tools/index_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/index_metadata.py b/tools/index_metadata.py index 5a1c2b58..6e0e80c1 100644 --- a/tools/index_metadata.py +++ b/tools/index_metadata.py @@ -117,11 +117,11 @@ def generate_os_host_list(hosts): def create_index(client, index, data): - return client.indices.create(index, body=json.dumps(data)) + return client.indices.create(index=index, body=data) def delete_index(client, index): return client.indices.delete(index=index, ignore=[400, 404]) -main() \ No newline at end of file +main() -- 2.34.1 From f2e01cf1ca558ae8a19e55b014aea6c67ba23af5 Mon Sep 17 00:00:00 2001 From: tischrei Date: Tue, 13 Jun 2023 09:11:03 +0000 Subject: [PATCH 07/16] finish index tool for document overview --- tools/index_metadata.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/index_metadata.py b/tools/index_metadata.py index 6e0e80c1..682515c8 100644 --- a/tools/index_metadata.py +++ b/tools/index_metadata.py @@ -117,7 +117,9 @@ def generate_os_host_list(hosts): def create_index(client, index, data): - return client.indices.create(index=index, body=data) + client.indices.create(index=index) + return client.index(index=index, body=data) + def delete_index(client, index): -- 2.34.1 From d21e492e1b07e993937374620453ed9e3f2953e2 Mon Sep 17 00:00:00 2001 From: SebastianGode Date: Tue, 13 Jun 2023 11:17:29 +0200 Subject: [PATCH 08/16] Fix pep8 --- tools/index_metadata.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/index_metadata.py b/tools/index_metadata.py index 682515c8..9251b85b 100644 --- a/tools/index_metadata.py +++ b/tools/index_metadata.py @@ -2,7 +2,6 @@ import otc_metadata import argparse import logging from opensearchpy import OpenSearch -import json metadata = otc_metadata.Services() @@ -121,7 +120,6 @@ def create_index(client, index, data): return client.index(index=index, body=data) - def delete_index(client, index): return client.indices.delete(index=index, ignore=[400, 404]) -- 2.34.1 From a3a3c5cd1fca2a8fe12a066f0dbfe5838f731bca Mon Sep 17 00:00:00 2001 From: SebastianGode Date: Tue, 20 Jun 2023 10:38:09 +0200 Subject: [PATCH 09/16] Output change --- otc_metadata/services.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/otc_metadata/services.py b/otc_metadata/services.py index f62ca3c0..95c2492f 100644 --- a/otc_metadata/services.py +++ b/otc_metadata/services.py @@ -115,12 +115,12 @@ class Services(object): def service_types_with_doc_types(self, environment=None): """Retrieve type and title from services and corresponding docs. - As well as a list of all doc_types. + As well as a list of all available doc types with title. :param str environment: Optional service environment. """ service_list = [] - doc_types = [] + docs = [] for service in self.all_services: if not service["service_title"]: @@ -139,8 +139,13 @@ class Services(object): "type": doc["type"] }) - if doc["type"] not in doc_types: - doc_types.append(doc["type"]) + new_doc = { + "type": doc["type"], + "title": doc["title"] + } + type_exists = any(doc_dict["type"] == new_doc["type"] for doc_dict in docs) + if not type_exists: + docs.append(new_doc) service_list.append({ "service_title": service["service_title"], @@ -150,7 +155,7 @@ class Services(object): res = { "services": service_list, - "doc_types": doc_types + "docs": docs } return res -- 2.34.1 From 2ed3bc4bf5be468d27ce963a4cca0d1154d2947b Mon Sep 17 00:00:00 2001 From: SebastianGode Date: Tue, 20 Jun 2023 10:38:47 +0200 Subject: [PATCH 10/16] Fix pep8 --- otc_metadata/services.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/otc_metadata/services.py b/otc_metadata/services.py index 95c2492f..5c7111f1 100644 --- a/otc_metadata/services.py +++ b/otc_metadata/services.py @@ -143,7 +143,9 @@ class Services(object): "type": doc["type"], "title": doc["title"] } - type_exists = any(doc_dict["type"] == new_doc["type"] for doc_dict in docs) + type_exists = any( + doc_dict["type"] == new_doc["type"] for doc_dict in docs + ) if not type_exists: docs.append(new_doc) -- 2.34.1 From 3e3e86ceca03ea7c1284bbd033f2a35422b6d99d Mon Sep 17 00:00:00 2001 From: SebastianGode Date: Fri, 23 Jun 2023 11:47:26 +0200 Subject: [PATCH 11/16] Filtering --- tools/index_metadata.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tools/index_metadata.py b/tools/index_metadata.py index 9251b85b..15078e7c 100644 --- a/tools/index_metadata.py +++ b/tools/index_metadata.py @@ -22,6 +22,11 @@ def parse_args(): help='Option deletes old index with the same name and creates new ' 'one.' ) + parser.add_argument( + '--all-doc-types', + action='store_true', + help='Upload all doc-types instead of only umn, api-ref and dev' + ) parser.add_argument( '--hosts', metavar='', @@ -62,7 +67,8 @@ def main(): logging.debug("Obtaining data from otc_metadata") data = getData( - environment=args.target_environment + environment=args.target_environment, + all_doc_types=args.all_doc_types ) logging.debug("Indexing data into OpenSearch") @@ -76,10 +82,16 @@ def main(): ) -def getData(environment): - return metadata.service_types_with_doc_types( +def getData(environment, all_doc_types): + metadatadata = metadata.service_types_with_doc_types( environment=environment ) + if not all_doc_types: + for i, doc in enumerate(metadatadata["docs"], 0): + if doc["type"] not in ["umn", "api-ref", "dev"]: + metadatadata["docs"].pop(i) + + return metadatadata def indexData(deleteIndex, hosts, index, username, password, data): -- 2.34.1 From 151133ff31d78bd412cbf79f370863d445c03fa4 Mon Sep 17 00:00:00 2001 From: SebastianGode Date: Fri, 23 Jun 2023 11:50:15 +0200 Subject: [PATCH 12/16] Debug --- tools/index_metadata.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/index_metadata.py b/tools/index_metadata.py index 15078e7c..f8ca5987 100644 --- a/tools/index_metadata.py +++ b/tools/index_metadata.py @@ -27,6 +27,11 @@ def parse_args(): action='store_true', help='Upload all doc-types instead of only umn, api-ref and dev' ) + parser.add_argument( + '--debug', + action='store_true', + help='Enable Debug mode' + ) parser.add_argument( '--hosts', metavar='', @@ -62,7 +67,8 @@ def parse_args(): def main(): - logging.basicConfig(level=logging.DEBUG) + if args.debug: + logging.basicConfig(level=logging.DEBUG) args = parse_args() logging.debug("Obtaining data from otc_metadata") @@ -86,11 +92,12 @@ def getData(environment, all_doc_types): metadatadata = metadata.service_types_with_doc_types( environment=environment ) + print(all_doc_types) if not all_doc_types: for i, doc in enumerate(metadatadata["docs"], 0): if doc["type"] not in ["umn", "api-ref", "dev"]: metadatadata["docs"].pop(i) - + print(metadatadata) return metadatadata -- 2.34.1 From 7e6a69ad28de60d925ed68ed29204b1e455e243a Mon Sep 17 00:00:00 2001 From: SebastianGode Date: Fri, 23 Jun 2023 11:50:42 +0200 Subject: [PATCH 13/16] Error fix --- tools/index_metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/index_metadata.py b/tools/index_metadata.py index f8ca5987..2bdb12c4 100644 --- a/tools/index_metadata.py +++ b/tools/index_metadata.py @@ -66,10 +66,11 @@ def parse_args(): def main(): + + args = parse_args() if args.debug: logging.basicConfig(level=logging.DEBUG) - args = parse_args() logging.debug("Obtaining data from otc_metadata") data = getData( -- 2.34.1 From 2dbeaff11318d4fcbc34c96c43f9a7dcd149e2dc Mon Sep 17 00:00:00 2001 From: SebastianGode Date: Fri, 23 Jun 2023 11:55:06 +0200 Subject: [PATCH 14/16] Function --- tools/index_metadata.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tools/index_metadata.py b/tools/index_metadata.py index 2bdb12c4..179d84ae 100644 --- a/tools/index_metadata.py +++ b/tools/index_metadata.py @@ -88,6 +88,10 @@ def main(): data=data ) +def filter_docs(metadata): + allowed_types = ["umn", "api-ref", "dev"] + metadata['docs'] = [doc for doc in metadata['docs'] if doc['type'] in allowed_types] + return metadata def getData(environment, all_doc_types): metadatadata = metadata.service_types_with_doc_types( @@ -95,11 +99,9 @@ def getData(environment, all_doc_types): ) print(all_doc_types) if not all_doc_types: - for i, doc in enumerate(metadatadata["docs"], 0): - if doc["type"] not in ["umn", "api-ref", "dev"]: - metadatadata["docs"].pop(i) - print(metadatadata) - return metadatadata + final_data = filter_docs(metadatadata) + print(final_data) + return final_data def indexData(deleteIndex, hosts, index, username, password, data): -- 2.34.1 From a236c3a34c54dc102e467fa4e827874044e57141 Mon Sep 17 00:00:00 2001 From: SebastianGode Date: Fri, 23 Jun 2023 11:58:04 +0200 Subject: [PATCH 15/16] Fix pep8 --- tools/index_metadata.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/index_metadata.py b/tools/index_metadata.py index 179d84ae..34214635 100644 --- a/tools/index_metadata.py +++ b/tools/index_metadata.py @@ -66,7 +66,7 @@ def parse_args(): def main(): - + args = parse_args() if args.debug: @@ -88,19 +88,20 @@ def main(): data=data ) + def filter_docs(metadata): allowed_types = ["umn", "api-ref", "dev"] - metadata['docs'] = [doc for doc in metadata['docs'] if doc['type'] in allowed_types] + metadata['docs'] = [doc for doc in metadata['docs'] + if doc['type'] in allowed_types] return metadata + def getData(environment, all_doc_types): metadatadata = metadata.service_types_with_doc_types( environment=environment ) - print(all_doc_types) if not all_doc_types: final_data = filter_docs(metadatadata) - print(final_data) return final_data -- 2.34.1 From 3dd766f0e938526206618dcd511d5bde3e2e26ce Mon Sep 17 00:00:00 2001 From: SebastianGode Date: Mon, 26 Jun 2023 10:00:03 +0200 Subject: [PATCH 16/16] Fixed issue --- tools/index_metadata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/index_metadata.py b/tools/index_metadata.py index 34214635..12e4dd41 100644 --- a/tools/index_metadata.py +++ b/tools/index_metadata.py @@ -100,6 +100,7 @@ def getData(environment, all_doc_types): metadatadata = metadata.service_types_with_doc_types( environment=environment ) + final_data = metadatadata if not all_doc_types: final_data = filter_docs(metadatadata) return final_data -- 2.34.1