diff --git a/tools/sql_keywords.py b/tools/sql_keywords.py new file mode 100644 index 00000000..4d2ac5a9 --- /dev/null +++ b/tools/sql_keywords.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- + +########################################################################## +# +# pgAdmin 4 - PostgreSQL Tools +# +# Copyright (C) 2013 - 2019, The pgAdmin Development Team +# This software is released under the PostgreSQL Licence +# +########################################################################## + +# This utility will extract SQL keywords from postgres website and pgsql keywords from +# the code git paths mentioned in PG_CODES_URLS +# Note that, PG_CODES_URLS may need to be changed manually per version change + +from __future__ import print_function +import re +import subprocess +import sys + +# required for OpenSSL - https sites +subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'requests', 'requests[security]']) +import requests + +PG_CODES_URLS = [ + "https://raw.github.com/postgres/postgres/REL_11_2/src/pl/plpgsql/src/pl_scanner.c", +] +PG_CODES_REGEX = "PG_KEYWORD\(\"([a-z]*)\"[A-Z_, ]*\)" + +PG_DOCS_URL = "https://www.postgresql.org/docs/current/sql-keywords-appendix.html" +PG_DOCS_REGEX = "<[a-z =\"]*>([A-Z_]*)" + + +def extract_keywords(text, reqex): + keywords = re.findall(reqex, text) + return [k.lower() for k in keywords] + + +def get_file_from_url(url): + req = requests.get(url) + return req.text + + +def get_keywords_pg_code(file_urls=PG_CODES_URLS, keyword_regex=PG_CODES_REGEX): + keywords = [] + for file_url in file_urls: + respText = get_file_from_url(file_url) + # Sample entry - PG_KEYWORD("begin", K_BEGIN, RESERVED_KEYWORD) + keywords.extend(extract_keywords(respText, keyword_regex)) + + return keywords + + +def get_keywords_pg_docs(docs_url=PG_DOCS_URL, keyword_regex=PG_DOCS_REGEX): + respText = get_file_from_url(docs_url) + # Sample entry - ABORT + keywords = extract_keywords(respText, keyword_regex) + + return keywords + + +def get_all_keywords(): + final_keywords = set() + + final_keywords.update(get_keywords_pg_code()) + final_keywords.update(get_keywords_pg_docs()) + + return len(final_keywords), " ".join(sorted(list(final_keywords))) + + +if __name__ == '__main__': + print("Started keywords extraction") + + total, keywords = get_all_keywords() + print("Total[{0}]".format(total)) + print("Keywords[{0}]".format(keywords))