make builder a click script

7 files changed, 84 insertions, 38 deletions

diff --git a/megapixels/app/builder/builder.py b/megapixels/app/builder/builder.py
index 620fc710..d1f882d6 100644
--- a/megapixels/app/builder/builder.py
+++ b/megapixels/app/builder/builder.py
@@ -1,18 +1,16 @@
 #!/usr/bin/python
 
-from dotenv import load_dotenv
-load_dotenv()
-
 import os
 import glob
 from jinja2 import Environment, FileSystemLoader, select_autoescape
 
-import s3
-import parser
-from paths import *
+import app.settings.app_cfg as cfg
+
+import app.builder.s3
+import app.builder.parser
 
 env = Environment(
-  loader=FileSystemLoader(template_path),
+  loader=FileSystemLoader(cfg.DIR_SITE_TEMPLATES),
   autoescape=select_autoescape([])
 )
 
@@ -26,7 +24,7 @@ def build_page(fn, research_posts):
   print(metadata['url'])
 
   dirname = os.path.dirname(fn)
-  output_path = public_path + metadata['url']
+  output_path = cfg.DIR_SITE_PUBLIC + metadata['url']
   output_fn = os.path.join(output_path, "index.html")
 
   skip_h1 = False
@@ -40,9 +38,9 @@ def build_page(fn, research_posts):
     template = env.get_template("page.html")
 
   if 'datasets/' in fn:
-    s3_dir = s3_datasets_path
+    s3_dir = cfg.S3_DATASETS_PATH
   else:
-    s3_dir = s3_site_path
+    s3_dir = cfg.S3_SITE_PATH
 
   s3_path = s3.make_s3_path(s3_dir, metadata['path'])
 
@@ -67,7 +65,7 @@ def build_page(fn, research_posts):
 def build_research_index(research_posts):
   metadata, sections = parser.read_metadata('../site/content/research/index.md')
   template = env.get_template("page.html")
-  s3_path = s3.make_s3_path(s3_site_path, metadata['path'])
+  s3_path = s3.make_s3_path(cfg.S3_SITE_PATH, metadata['path'])
   content = parser.parse_markdown(sections, s3_path, skip_h1=False)
   content += parser.parse_research_index(research_posts)
   html = template.render(
@@ -82,9 +80,6 @@ def build_research_index(research_posts):
 
 def build_site():
   research_posts = parser.read_research_post_index()
-  for fn in glob.iglob(os.path.join(content_path, "**/*.md"), recursive=True):
+  for fn in glob.iglob(os.path.join(cfg.DIR_SITE_CONTENT, "**/*.md"), recursive=True):
     build_page(fn, research_posts)
   build_research_index(research_posts)
-
-if __name__ == '__main__':
-  build_site()
diff --git a/megapixels/app/builder/parser.py b/megapixels/app/builder/parser.py
index dd3643bf..ea84f658 100644
--- a/megapixels/app/builder/parser.py
+++ b/megapixels/app/builder/parser.py
@@ -3,8 +3,8 @@ import re
 import glob
 import mistune
 
-import s3
-from paths import *
+import app.settings.app_cfg
+import app.builder.s3
 
 renderer = mistune.Renderer(escape=False)
 markdown = mistune.Markdown(renderer=renderer)
@@ -70,7 +70,7 @@ def parse_markdown(sections, s3_path, skip_h1=False):
 def parse_research_index(research_posts):
   content = "<div class='research_index'>"
   for post in research_posts:
-    s3_path = s3.make_s3_path(s3_site_path, post['path'])
+    s3_path = s3.make_s3_path(app_cfg.S3_SITE_PATH, post['path'])
     if 'image' in post:
       post_image = s3_path + post['image']
     else:
diff --git a/megapixels/app/builder/paths.py b/megapixels/app/builder/paths.py
deleted file mode 100644
index 356f2f3d..00000000
--- a/megapixels/app/builder/paths.py
+++ /dev/null
@@ -1,6 +0,0 @@
-
-s3_site_path = "v1/site"
-s3_datasets_path = "v1" # datasets is already in the filename
-public_path = "../site/public"
-content_path = "../site/content"
-template_path = "../site/templates"
diff --git a/megapixels/app/builder/s3.py b/megapixels/app/builder/s3.py
index 41ecdf61..99726a4d 100644
--- a/megapixels/app/builder/s3.py
+++ b/megapixels/app/builder/s3.py
@@ -1,17 +1,6 @@
 import os
 import glob
 import boto3
-from paths import *
-
-session = boto3.session.Session()
-
-s3_client = session.client(
-  service_name='s3',
-  aws_access_key_id=os.getenv('S3_KEY'),
-  aws_secret_access_key=os.getenv('S3_SECRET'),
-  endpoint_url=os.getenv('S3_ENDPOINT'),
-  region_name=os.getenv('S3_REGION'),
-)
 
 def sync_directory(base_fn, s3_path, metadata):
   fns = {}
@@ -23,6 +12,16 @@ def sync_directory(base_fn, s3_path, metadata):
 
   remote_path = s3_path + metadata['url']
 
+  session = boto3.session.Session()
+
+  s3_client = session.client(
+    service_name='s3',
+    aws_access_key_id=os.getenv('S3_KEY'),
+    aws_secret_access_key=os.getenv('S3_SECRET'),
+    endpoint_url=os.getenv('S3_ENDPOINT'),
+    region_name=os.getenv('S3_REGION'),
+  )
+
   directory = s3_client.list_objects(Bucket=os.getenv('S3_BUCKET'), Prefix=remote_path)
   prefixes = []
 
diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py
index 1d3fbc4c..9e03357b 100644
--- a/megapixels/app/settings/app_cfg.py
+++ b/megapixels/app/settings/app_cfg.py
@@ -70,7 +70,6 @@ load_dotenv(dotenv_path=DIR_DOTENV)
 DIR_ASSETS = join(DIR_APP, 'assets')
 FP_FONT = join(DIR_ASSETS, 'font')
 
-
 # -----------------------------------------------------------------------------
 # click chair settings
 # -----------------------------------------------------------------------------
@@ -91,7 +90,6 @@ CKPT_ZERO_PADDING = 9
 HASH_TREE_DEPTH = 3
 HASH_BRANCH_SIZE = 3
 
-
 DLIB_FACEREC_JITTERS = 5  # number of face recognition jitters
 DLIB_FACEREC_PADDING = 0.25  # default dlib
 
@@ -120,9 +118,18 @@ reset: Clear all formatting (both foreground and background colors).
 """
 LOGFILE_FORMAT = "%(log_color)s%(levelname)-8s%(reset)s %(cyan)s%(filename)s:%(lineno)s:%(bold_cyan)s%(funcName)s() %(reset)s%(message)s"
 
-
 # -----------------------------------------------------------------------------
 # S3 storage
 # -----------------------------------------------------------------------------
 S3_MEDIA_ROOT = 's3://megapixels/v1/media/'
 S3_METADATA_ROOT = 's3://megapixels/v1/metadata/'
+
+# -----------------------------------------------------------------------------
+# Builder
+# -----------------------------------------------------------------------------
+
+S3_SITE_PATH = "v1/site"
+S3_DATASETS_PATH = "v1" # datasets is already in the filename
+DIR_SITE_PUBLIC = "../site/public"
+DIR_SITE_CONTENT = "../site/content"
+DIR_SITE_TEMPLATES = "../site/templates"
diff --git a/megapixels/cli_builder.py b/megapixels/cli_builder.py
new file mode 100644
index 00000000..8bc814f1
--- /dev/null
+++ b/megapixels/cli_builder.py
@@ -0,0 +1,36 @@
+# --------------------------------------------------------
+# add/edit commands in commands/faiss directory
+# --------------------------------------------------------
+
+import click
+
+from app.settings import app_cfg as cfg
+from app.utils import logger_utils
+from app.models.click_factory import ClickSimple
+
+# click cli factory
+cc = ClickSimple.create(cfg.DIR_COMMANDS_BUILDER)
+
+# --------------------------------------------------------
+# CLI
+# --------------------------------------------------------
+@click.group(cls=cc, chain=False)
+@click.option('-v', '--verbose', 'verbosity', count=True, default=4, 
+  show_default=True,
+  help='Verbosity: -v DEBUG, -vv INFO, -vvv WARN, -vvvv ERROR, -vvvvv CRITICAL')
+@click.pass_context
+def cli(ctx, **kwargs):
+  """\033[1m\033[94mMegaPixels: FAISS Data Scripts\033[0m                                                
+  """
+  ctx.opts = {}
+  # init logger
+  logger_utils.Logger.create(verbosity=kwargs['verbosity'])
+
+
+
+# --------------------------------------------------------
+# Entrypoint
+# --------------------------------------------------------
+if __name__ == '__main__':
+    cli()
+
diff --git a/megapixels/commands/builder/build.py b/megapixels/commands/builder/build.py
new file mode 100644
index 00000000..fc4fb302
--- /dev/null
+++ b/megapixels/commands/builder/build.py
@@ -0,0 +1,15 @@
+"""
+Build the static site
+"""
+
+import click
+
+from app.builder.builder import build_site
+
+@click.command()
+@click.pass_context
+def cli(ctx):
+  """Build the static site
+  """
+  print('Building the site...')
+  build_site()