summaryrefslogtreecommitdiff
path: root/megapixels
diff options
context:
space:
mode:
Diffstat (limited to 'megapixels')
-rw-r--r--megapixels/app/settings/app_cfg.py1
-rw-r--r--megapixels/app/settings/paths.py163
-rw-r--r--megapixels/app/settings/types.py12
-rw-r--r--megapixels/app/utils/path_utils.py34
4 files changed, 45 insertions, 165 deletions
diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py
index 0507366f..9ea4b72b 100644
--- a/megapixels/app/settings/app_cfg.py
+++ b/megapixels/app/settings/app_cfg.py
@@ -23,6 +23,7 @@ HaarCascadeVar = click_utils.ParamVar(types.HaarCascade)
LogLevelVar = click_utils.ParamVar(types.LogLevel)
MetadataVar = click_utils.ParamVar(types.Metadata)
DatasetVar = click_utils.ParamVar(types.Dataset)
+DataStoreVar = click_utils.ParamVar(types.DataStore)
# # data_store
DATA_STORE = '/data_store_hdd/'
diff --git a/megapixels/app/settings/paths.py b/megapixels/app/settings/paths.py
deleted file mode 100644
index bc1333ba..00000000
--- a/megapixels/app/settings/paths.py
+++ /dev/null
@@ -1,163 +0,0 @@
-import os
-from os.path import join
-import logging
-
-from vframe.settings import vframe_cfg as vcfg
-from vframe.settings import types
-
-class Paths:
-
- # class properties
- MAPPINGS_DATE = vcfg.SUGARCUBE_DATES[0]
- DIR_APP_VFRAME = 'apps/vframe/'
- DIR_APP_SA = 'apps/syrianarchive'
- DIR_MODELS_VFRAME = join(DIR_APP_VFRAME, 'models')
- DIR_DARKNET = join(DIR_MODELS_VFRAME, 'darknet/pjreddie')
- DIR_DARKNET_VFRAME = join(DIR_MODELS_VFRAME, 'darknet/vframe')
- DIR_MEDIA = join(DIR_APP_SA, 'media')
- DIR_METADATA = join(DIR_APP_SA, 'metadata')
- DIR_RECORDS = join(DIR_APP_SA, 'records')
- DIR_REPORTS = join(DIR_APP_SA, 'reports')
-
-
- def __init__(self):
- pass
-
- @classmethod
- def DataStorePath(cls, data_store=types.DataStore.HDD):
- return '/data_store_{}'.format(data_store.name.lower())
-
- # -------------------------------------------------------------------------------
- # Darknet Paths
-
- @classmethod
- def darknet_classes(cls, data_store=types.DataStore.HDD, opt_net=types.DetectorNet.COCO):
- if opt_net == types.DetectorNet.COCO:
- fp = join(cls.DIR_DARKNET, 'data', 'coco.names')
- elif opt_net == types.DetectorNet.COCO_SPP:
- fp = join(cls.DIR_DARKNET, 'data', 'coco.names')
- elif opt_net == types.DetectorNet.VOC:
- fp = join(cls.DIR_DARKNET, 'data', 'voc.names')
- elif opt_net == types.DetectorNet.OPENIMAGES:
- fp = join(cls.DIR_DARKNET, 'data', 'openimages.names')
- elif opt_net == types.DetectorNet.SUBMUNITION:
- fp = join(cls.DIR_DARKNET_VFRAME, 'munitions_09b', 'classes.txt')
- return join(cls.DataStorePath(data_store), fp)
-
- @classmethod
- def darknet_data(cls, data_store=types.DataStore.HDD, opt_net=types.DetectorNet.COCO, as_bytes=True):
- if opt_net == types.DetectorNet.COCO:
- fp = join(cls.DIR_DARKNET, 'cfg', 'coco.data')
- elif opt_net == types.DetectorNet.COCO_SPP:
- fp = join(cls.DIR_DARKNET, 'cfg', 'coco.data')
- elif opt_net == types.DetectorNet.VOC:
- fp = join(cls.DIR_DARKNET, 'cfg', 'voc.data')
- elif opt_net == types.DetectorNet.OPENIMAGES:
- fp = join(cls.DIR_DARKNET, 'cfg', 'openimages.data')
- elif opt_net == types.DetectorNet.SUBMUNITION:
- fp = join(cls.DIR_DARKNET_VFRAME, 'munitions_09b', 'meta.data')
- fp = join(cls.DataStorePath(data_store), fp)
- if as_bytes:
- return bytes(fp, encoding="utf-8")
- else:
- return fp
-
-
- @classmethod
- def darknet_cfg(cls, data_store=types.DataStore.HDD, opt_net=types.DetectorNet.COCO, as_bytes=True):
- if opt_net == types.DetectorNet.COCO:
- fp = join(cls.DIR_DARKNET, 'cfg', 'yolov3.cfg')
- elif opt_net == types.DetectorNet.COCO_SPP:
- fp = join(cls.DIR_DARKNET, 'cfg', 'yolov3-spp.cfg')
- elif opt_net == types.DetectorNet.VOC:
- fp = join(cls.DIR_DARKNET, 'cfg', 'yolov3-voc.cfg')
- elif opt_net == types.DetectorNet.OPENIMAGES:
- fp = join(cls.DIR_DARKNET, 'cfg', 'yolov3-openimages.cfg')
- elif opt_net == types.DetectorNet.SUBMUNITION:
- fp = join(cls.DIR_DARKNET_VFRAME, 'munitions_09b', 'yolov3.cfg')
- fp = join(cls.DataStorePath(data_store), fp)
- if as_bytes:
- return bytes(fp, encoding="utf-8")
- else:
- return fp
-
- @classmethod
- def darknet_weights(cls, data_store=types.DataStore.HDD, opt_net=types.DetectorNet.COCO, as_bytes=True):
- if opt_net == types.DetectorNet.COCO:
- fp = join(cls.DIR_DARKNET, 'weights', 'yolov3.weights')
- elif opt_net == types.DetectorNet.COCO_SPP:
- fp = join(cls.DIR_DARKNET, 'weights', 'yolov3-spp.weights')
- elif opt_net == types.DetectorNet.VOC:
- fp = join(cls.DIR_DARKNET, 'weights', 'yolov3-voc.weights')
- elif opt_net == types.DetectorNet.OPENIMAGES:
- fp = join(cls.DIR_DARKNET, 'weights', 'yolov3-openimages.weights')
- elif opt_net == types.DetectorNet.SUBMUNITION:
- fp = join(cls.DIR_DARKNET_VFRAME, 'munitions_09b/weights', 'yolov3_40000.weights')
- fp = join(cls.DataStorePath(data_store), fp)
- if as_bytes:
- return bytes(fp, encoding="utf-8")
- else:
- return fp
-
- # -------------------------------------------------------------------------------
- # Metadata Paths
-
- @classmethod
- def mapping_index(cls, opt_date, data_store=types.DataStore.HDD, verified=types.Verified.VERIFIED,
- file_format=types.FileExt.PKL):
- """Returns filepath to a mapping file. Mapping files are the original Suguarcube mapping data"""
- fname = 'index.pkl' if file_format == types.FileExt.PKL else 'index.json'
- # data_store = 'data_store_{}'.format(data_store.name.lower())
- date_str = opt_date.name.lower()
- fp = join(cls.DataStorePath(data_store), cls.DIR_METADATA, 'mapping', date_str, verified.name.lower(), fname)
- return fp
-
- @classmethod
- def media_record_index(cls, data_store=types.DataStore.HDD, verified=types.Verified.VERIFIED,
- file_format=types.FileExt.PKL):
- """Returns filepath to a mapping file. Mapping files are the original Suguarcube mapping data"""
- fname = 'index.pkl' if file_format == types.FileExt.PKL else 'index.json'
- metadata_type = types.Metadata.MEDIA_RECORD.name.lower()
- fp = join(cls.DataStorePath(data_store), cls.DIR_METADATA, metadata_type, verified.name.lower(), fname)
- return fp
-
- @classmethod
- def metadata_index(cls, metadata_type, data_store=types.DataStore.HDD,
- verified=types.Verified.VERIFIED, file_format=types.FileExt.PKL):
- """Uses key from enum to get folder name and construct filepath"""
- fname = 'index.pkl' if file_format == types.FileExt.PKL else 'index.json'
- fp = join(cls.DataStorePath(data_store), cls.DIR_METADATA, metadata_type.name.lower(),
- verified.name.lower(), fname)
- return fp
-
- @classmethod
- def metadata_dir(cls, metadata_type, data_store=types.DataStore.HDD, verified=types.Verified.VERIFIED):
- """Uses key from enum to get folder name and construct filepath"""
- fp = join(cls.DataStorePath(data_store), cls.DIR_METADATA, metadata_type.name.lower(),
- verified.name.lower())
- return fp
-
- @classmethod
- def metadata_tree_dir(cls, metadata_type, data_store=types.DataStore.HDD):
- """Uses key from enum to get folder name and construct filepath"""
- fp = join(cls.DataStorePath(data_store), cls.DIR_METADATA, metadata_type.name.lower())
- return fp
-
- @classmethod
- def media_dir(cls, media_type, data_store=types.DataStore.HDD, verified=types.Verified.VERIFIED):
- """Returns the directory path to a media directory"""
- fp = join(cls.DataStorePath(data_store), cls.DIR_MEDIA, media_type.name.lower(), verified.name.lower())
- return fp
-
- # @classmethod
- # def keyframe(cls, dir_media, idx, image_size=types.ImageSize.MEDIUM):
- # """Returns path to keyframe image using supplied cls.media directory"""
- # idx = str(idx).zfill(vcfg.ZERO_PADDING)
- # size_label = vcfg.IMAGE_SIZE_LABELS[image_size]
- # fp = join(dir_media, sha256_tree, sha256, idx, size_label, 'index.jpg')
- # return fp
-
- @classmethod
- def dnn(cls):
- """Returns configurations for available DNNs"""
- pass \ No newline at end of file
diff --git a/megapixels/app/settings/types.py b/megapixels/app/settings/types.py
index 7157436d..4f403846 100644
--- a/megapixels/app/settings/types.py
+++ b/megapixels/app/settings/types.py
@@ -22,7 +22,15 @@ class CVTarget(Enum):
class HaarCascade(Enum):
FRONTAL, ALT, ALT2, PROFILE = range(4)
-
+
+# ---------------------------------------------------------------------
+# Storage
+# --------------------------------------------------------------------
+
+class DataStore(Enum):
+ """Storage devices. Paths are symlinked to root (eg /data_store_nas)"""
+ NAS, HDD, SSD = range(3)
+
# ---------------------------------------------------------------------
# Logger, monitoring
# --------------------------------------------------------------------
@@ -37,7 +45,7 @@ class LogLevel(Enum):
# --------------------------------------------------------------------
class Metadata(Enum):
- IDENTITIES, POSES, ROIS, FILE_META, SHAS, UUIDS, FACE_VECTORS = range(7)
+ IDENTITY, FILEPATH, SHA256, UUID, FACE_VECTOR, FACE_POSE, FACE_ROI = range(7)
class Dataset(Enum):
LFW, VGG_FACE2 = range(2)
diff --git a/megapixels/app/utils/path_utils.py b/megapixels/app/utils/path_utils.py
new file mode 100644
index 00000000..ddbc2864
--- /dev/null
+++ b/megapixels/app/utils/path_utils.py
@@ -0,0 +1,34 @@
+import os
+from os.path import join
+import logging
+
+from app.settings import app_cfg as cfg
+from app.settings import types
+
+
+# class properties
+DATA_STORE = cfg.DATA_STORE
+DIR_DATSETs = cfg.DIR_DATSET_NAS
+DIR_DATASETS = join(DATA_STORE, 'datasets')
+DIR_PEOPLE = 'datasets/people'
+
+# -------------------------------------------------------------------------
+# Metadata
+# -------------------------------------------------------------------------
+
+def data_store_dir(opt_data_store):
+ return f'/data_store_{opt_data_store.name.lower()}'
+
+def dataset_dir(opt_dataset):
+ return f'{opt_dataset.name.lower()}'
+
+def metadata_dir(opt_data_store, opt_dataset):
+ return join(data_store_dir(opt_data_store), DIR_PEOPLE, dataset_dir(opt_dataset), 'metadata')
+
+def metadata(opt_data_store, opt_dataset, opt_metadata):
+ return join(metadata_dir(opt_data_store, opt_dataset), f'{opt_metadata.name.lower()}.csv')
+
+
+# -------------------------------------------------------------------------
+# Media
+# ------------------------------------------------------------------------- \ No newline at end of file
generated by cgit v1.2.3-70-g09d2 (git 2.51.0) at 2025-12-10 14:59:40 +0000