From c7efa327f2968638ddfe77636bdeccb84cf065e0 Mon Sep 17 00:00:00 2001 From: yo mama Date: Wed, 11 Nov 2015 15:44:10 -0800 Subject: first --- scripts/log_loader.py | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 scripts/log_loader.py (limited to 'scripts') diff --git a/scripts/log_loader.py b/scripts/log_loader.py new file mode 100644 index 0000000..5b2e1d6 --- /dev/null +++ b/scripts/log_loader.py @@ -0,0 +1,79 @@ +#!/usr/bin/python2 +# coding: utf-8 +import sys +import os +import re +import time +from sqlalchemy.orm import sessionmaker +from photoblaster.db.s3db import S3 as Db + +LOGS_PATH = "/mnt/sdc1/s3LOGS" + + +DB_HOST = "localhost" +DB_USER = "asdfus" +DB_PASSWORD = "gTYgT&M6q" +DB_NAME = "asdfus" + +db = Db() + +class s3LogReader(object): + def __init__(self): + self.successes = [] + + def list_logdir(self): + return os.listdir(LOGS_PATH) + + def _logfile_fullpath(self, logfile): + return os.path.join(LOGS_PATH, logfile) + + def log_success(self, logfile, remove=False): + self.success_log.write("%s\n" % logfile) + logfile_path = self._logfile_fullpath(logfile) + if remove: + try: + os.unlink(logfile_path) + except Exception as e: + sys.stderr.write("Unable to unlink %s\n" % logfile_path) + sys.stderr.write("%s\n" % e ) + self.success_log.close() + sys.exit(1) + + def log_error(self, logfile): + sys.stderr.write("Unable to process logfile: %s\n" % logfile) + self.failure_log.write("%s\n" % logfile) + + def read_log(self, logfile): + if logfile in self.successes: + sys.stderr.write("it's in successes already...returning\n") + return + logfile_path = self._logfile_fullpath(logfile) + f = open(logfile_path, "r") + data = f.readlines() + for line in data: + line = re.sub(r'\[', "", line) + parts = re.split(r'\s', line) + timestamp = parts[2] + file_address = parts[8] + if re.match(r'.*im/', file_address): + timestamp = time.strptime("08/Sep/2014:19:58:11", + "%d/%b/%Y:%H:%M:%S") + timestamp = int(time.mktime(timestamp)) + if db.update_existing(last_accessed=timestamp, address=file_address, logfile=logfile): + continue + else: + db.insert_new_from_log(last_accessed=timestamp, address=file_address) + continue + + def process_logdir(self): + logs = self.list_logdir() + for log in logs: + try: + self.read_log(log) + self.log_success(log) + except Exception as e: + sys.stderr.write("%s\n" % e ) + +if __name__ == "__main__": + logreader = s3LogReader() + logreader.process_logdir() -- cgit v1.2.3-70-g09d2 From aefb9559287130e55ab6536e97cce8bbafdca04e Mon Sep 17 00:00:00 2001 From: pepperpepperpepper Date: Sun, 15 Nov 2015 20:01:54 -0800 Subject: ok looking better...getting ready for merge --- scripts/remote_host_test.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 scripts/remote_host_test.py (limited to 'scripts') diff --git a/scripts/remote_host_test.py b/scripts/remote_host_test.py new file mode 100644 index 0000000..c38e3c7 --- /dev/null +++ b/scripts/remote_host_test.py @@ -0,0 +1,26 @@ +from flask import request +from flask import jsonify +from flask import Flask +import re +app = Flask(__name__) + +@app.route("/") +def hello(): + return "Hello World!" + +@app.route("/testme", methods=["GET"]) +def get_my_ip(): + host = request.headers.getlist("X-Forwarded-For")[0] + regex = re.compile(r'[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+') + forwarded_ip = regex.search(host).group() + return jsonify( + { + 'ip1': request.remote_addr, + 'ip2': request.environ['REMOTE_ADDR'], + 'ip3': request.environ.get('HTTP_HOST', "nothing yet"), + 'ip4': forwarded_ip, + 'ip5': request.environ.get('HTTP_X_REAL_IP', request.remote_addr) + } + ), 200 +if __name__ == "__main__": + app.run(port=9994) -- cgit v1.2.3-70-g09d2