Added saving results

config.ini

@@ -3,13 +3,11 @@ name = MODTBGA
 bands = 31, 32
 11um = 31
 12um = 32
+calculate_radiance = 0
 
 [data]
-root = ../contraildetect/data/
-hdf = ../contraildetect/data/input/hdf
-flights = ../contraildetect/data/input/flights
-processed = ../contraildetect/processed
-tmp = ../contraildetect/data/tmp
+hdf = ../contraildetect/data/hdf
+flights = ../contraildetect/data/flights
 output = ../contraildetect/data/output
 
 [extraction]

config.py

@@ -24,10 +24,6 @@ def product_name():
     return cp.get(prod_key, 'name')
 
 
-def product_dir():
-    return os.path.join(hdf_input(), product_name())
-
-
 def bands():
     return [int(band.strip()) for band in cp.get(prod_key, 'bands').split(',')]
 
@@ -47,30 +43,23 @@ DIRECTORIES
 data_key = 'data'
 
 
-def hdf_input():
-    return cp.get(data_key, 'hdf')
+def product_dir():
+    return os.path.join(cp.get(data_key, 'hdf'), product_name())
 
 
 def flight_input():
     return cp.get(data_key, 'flights')
 
 
-def processed():
-    proc_dir = os.path.join(cp.get(data_key, 'processed'), _today())
-    if not os.path.exists(proc_dir):
-        os.makedirs(proc_dir)
-    return proc_dir
-
-
 def processed_flights():
-    proc_flight_dir = os.path.join(processed(), 'flights')
+    proc_flight_dir = os.path.join(flight_input(), 'processed')
     if not os.path.exists(proc_flight_dir):
         os.makedirs(proc_flight_dir)
     return proc_flight_dir
 
 
-def processed_img():
-    proc_img_dir = os.path.join(processed(), 'img')
+def output_img():
+    proc_img_dir = os.path.join(output_dir(), 'img')
     if not os.path.exists(proc_img_dir):
         os.makedirs(proc_img_dir)
     return proc_img_dir

main.py

 import os
+from glob import glob
+
+from tqdm import tqdm
 
 import config
-import pipeline
-from stages.flight_filtering import filter_flights
-from stages.prepare import create_job
+from constants import columns_to_save
+from stages import prepare, transformation, detection, flight_filtering, segmentation, assembling
 from util import check_and_repair_csv
-import pandas as pd
+
+tqdm.pandas()
 
 """
 INPUT
 """
-# input_files = glob(config.product_dir() + '/*.hdf')
-input_files = [os.path.join(config.product_dir(), 'MODTBGA.A2012094.h15v03.006.2015240000438.hdf')]
+input_files = glob(config.product_dir() + '/*.hdf')
+# input_files = [os.path.join(config.product_dir(), 'MODTBGA.A2012094.h15v03.006.2015240000438.hdf')]
 
 # TODO: collecting system information (cpu cores, current time)
 # TODO: multiprocessing
 # TODO: implement opt-out/in stages
 # TODO: improve logging
 
-job_df = create_job(input_files)
+print('Creating job')
+job_df = prepare.create_job(input_files)
 
 # TODO: print status
 
 """
 CHECK FLIGHT CSV
 """
+print('Checking flight files')
 check_and_repair_csv(job_df)
 
 # TODO: print status
@@ -32,14 +37,36 @@ check_and_repair_csv(job_df)
 """
 FILTER FLIGHT CSV
 """
-job_df = filter_flights(job_df)
+print('Filter flights')
+job_df = job_df.progress_apply(flight_filtering.filter_flights, axis=1)
 
 """
-RUN PIPELINE
+TRANSFORMATION
 """
-job_df = job_df.apply(pipeline.run, axis=1)
+print('Running CDA/thresholding')
+job_df = job_df.progress_apply(transformation.cda_preprocess, axis=1)
+job_df = job_df.progress_apply(transformation.threshold, axis=1)
 
 """
-RESULTS
+DETECTION
+"""
+print('Running detection')
+job_df = job_df.progress_apply(detection.probabilistic_hough, axis=1)
+
+"""
+SEGMENTATION
 """
+print('Running segmentation')
+job_df = job_df.progress_apply(segmentation.run, axis=1)
 
+"""
+ASSEMBLING INFORMATION
+"""
+print('Assembling information')
+job_df = job_df.progress_apply(assembling.collect, axis=1)
+
+"""
+RESULTS
+"""
+print('Saving results')
+job_df.to_csv(os.path.join(config.output_dir(), 'output.csv'), columns=columns_to_save, index=False)

stages/flight_filtering.py

@@ -50,7 +50,7 @@ def _filter_by_time_and_position(day_segment_df: pd.DataFrame, south: float, nor
     return computed_df
 
 
-def _filter_flights(row: pd.Series):
+def filter_flights(row: pd.Series):
     if row[col.flight_processed] is not None:
         flight_prc = pd_read_csv(row[col.flight_processed], DTYPES)
         row[col.flight_count] = len(flight_prc['FLIGHT_ID'].unique().tolist())
@@ -74,7 +74,3 @@ def _filter_flights(row: pd.Series):
         del flights_fil
 
         return row
-
-
-def filter_flights(job_df: pd.DataFrame):
-    return job_df.apply(_filter_flights, axis=1)

stages/prepare.py

@@ -50,7 +50,7 @@ def create_row(filepath: str):
             }
 
 
-def create_job(filepath_list):
+def create_job(filepath_list) -> pd.DataFrame:
     job_df = pd.DataFrame()
 
     for filepath in filepath_list: