Commit d4b65f07 authored by Julian Balling's avatar Julian Balling
Browse files

Runtime Evaluation Scripts

parent 184ec83b
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
#!/bin/bash
for mdl in Techem Ista Brunata Kalo BFW Minol Sonstige; do
for batch in alt.mdl neu.mdl; do
echo "GENERATING PATHMAP FOR $mdl"
test -d $batch/$mdl || continue;
# extract image data and convert it to COCO-Format -> keep Order and References
#ocrd-segment-extract-formdata -I OCR-D-OCR-TESS-deu-SEG-tesseract-sparse-FORM-COR -O OCR-D-OCR-TESS-deu-SEG-tesseract-sparse-FORM-IMG,OCR-D-OCR-TESS-deu-SEG-tesseract-sparse-FORM-COCO --overwrite
# create file path mappings from extracted images to originals
{ echo '{';
find $batch/$mdl -type f -path "*OCR-D-OCR-TESS-deu-SEG-tesseract-sparse-FORM-IMG/*.png" -printf "%P\n" | sed -e 's,^,",;s,$,",;h;s,^.*OCR-D-OCR-TESS-deu-SEG-tesseract-sparse-FORM-IMG_,"OCR-D-IMG-CROP/OCR-D-IMG-CROP_,;s/$/,/;x;G;s/\
/:/';
echo '"":""}'; } > $batch/$mdl/pathmap_runtime.json
maskrcnn-formdata --cwd merge --dataset $batch/$mdl/OCR-D-OCR-TESS-deu-SEG-tesseract-sparse-FORM-COCO/*.coco.json --split 0 --dataset-merged $batch/$mdl/merged.PRED_runtime.coco.json --replace-names $batch/$mdl/pathmap_runtime.json --rle --sort --combine
echo "PROCESSING DIRECTORY $batch/$mdl"
pushd $batch/$mdl
maskrcnn-formdata --cwd merge --dataset merged.GT_runtime.coco.json --split 0 --dataset-merged merged.GT_runtime.coco.json --rle --sort --combine
maskrcnn-formdata --cwd merge --dataset merged.PRED_runtime.coco.json --split 0 --dataset-merged merged.PRED_runtime.coco.json --rle --sort --combine
# Create merged.FULLPRED.coco.res.json for Visualization with COCO-Explorer
maskrcnn-formdata --cwd merge --dataset merged.PRED_runtime.coco.json --split 0 --dataset-merged merged.PRED_runtime.coco.res.json --rle --sort --combine --anns-only
# merge Datasets of each Class
# calculate Precision, Recall (COCO-Format)
maskrcnn-formdata compare --dataset merged.GT_runtime.coco.json --dataset-pred merged.PRED_runtime.coco.json | tee smarthec_evaluation_runtime.log
# now get only validation data:
maskrcnn-formdata --cwd merge --dataset merged.GT.coco.json --dataset-merged merged.VAL-GT_runtime.coco.json --replace-names pathmap_runtime.json --rle --sort --combine
maskrcnn-formdata --cwd merge --dataset OCR-D-OCR-TESS-deu-SEG-tesseract-sparse-FORM-COCO/*.coco.json --dataset-merged merged.VAL-PRED_runtime.coco.json --replace-names pathmap_runtime.json --rle --sort --combine
# sort datasets
maskrcnn-formdata --cwd merge --dataset merged.VAL-GT_runtime.coco.json --dataset-merged merged.VAL-GT_runtime.coco.json --rle --sort --combine
maskrcnn-formdata --cwd merge --dataset merged.VAL-PRED_runtime.coco.json --dataset-merged merged.VAL-PRED_runtime.coco.json --rle --sort --combine
# create merged.VAL-PRED_runtime.coco.res.json COCO Format
maskrcnn-formdata --cwd merge --dataset merged.VAL-PRED_runtime.coco.json --dataset-merged merged.VAL-PRED_runtime.coco.res.json --rle --sort --combine --anns-only
# compare
maskrcnn-formdata compare --dataset merged.VAL-GT_runtime.coco.json --dataset-pred merged.VAL-PRED_runtime.coco.json | tee smarthec_evaluation_val_runtime.log
popd
done
done
# concatenate all logs into final log file
#find . -name smarthec_evaluation_runtime.log -exec cat {} + > smarthec_evaluation_runtime_all.log
#!/bin/bash
savelog -c 100 -nlp predictions.csv
# parses smarthec_evaluation.log files into CSV files for precision and recall
echo "batch;mdl;category;precision;recall" > predictions.csv
find . -xdev -name "smarthec_evaluation_runtime.log" | sort | while read path; do origin=$(dirname ${path#./}); sed -n '/^BG/d;/prc/{N;s/\
/:/;s/ prc//;p}' $path | sed "s/^/${origin/\//:}:/"; done | cut -d: -f 1-4,6 | tr : ";" >> predictions.csv
# now without training data:
savelog -c 100 -nlp val-predictions.csv
# parses smarthec_evaluation_val.log files into CSV files for precision and recall
echo "batch;mdl;category;precision;recall" > val-predictions.csv
find . -xdev -name "smarthec_evaluation_val_runtime.log" | sort | while read path; do origin=$(dirname ${path#./}); sed -n '/^BG/d;/prc/{N;s/\
/:/;s/ prc//;p}' $path | sed "s/^/${origin/\//:}:/"; done | cut -d: -f 1-4,6 | tr : ";" >> val-predictions.csv
#!/bin/bash
# merge of all workspace-specific GT/PRED COCOs into single GT/PRED COCO,
# prefixing image file names, and keeping only annotations for pred (to feed into COCO explorer)
maskrcnn-formdata merge --dataset {alt.mdl/{Techem,Ista,Sonstige,Brunata},neu.mdl/{Techem,Ista,Sonstige,Brunata,Kalo,BFW,Minol}}/merged.GT_runtime.coco.json --split 0 --dataset-merged all.GT_runtime.coco.json --sort
maskrcnn-formdata merge --dataset {alt.mdl/{Techem,Ista,Sonstige,Brunata},neu.mdl/{Techem,Ista,Sonstige,Brunata,Kalo,BFW,Minol}}/merged.PRED_runtime.coco.json --split 0 --dataset-merged all.PRED_runtime.coco.json --sort
maskrcnn-formdata merge --dataset {alt.mdl/{Techem,Ista,Sonstige,Brunata},neu.mdl/{Techem,Ista,Sonstige,Brunata,Kalo,BFW,Minol}}/merged.PRED_runtime.coco.json --split 0 --dataset-merged all.PRED_runtime.coco.res.json --sort --anns-only
# now without training data:
maskrcnn-formdata merge --dataset {alt.mdl/{Techem,Ista,Sonstige,Brunata},neu.mdl/{Techem,Ista,Sonstige,Brunata,Kalo,BFW,Minol}}/merged.VAL-GT.coco.json --split 0 --dataset-merged all.VAL-GT.coco.json --sort
maskrcnn-formdata merge --dataset {alt.mdl/{Techem,Ista,Sonstige,Brunata},neu.mdl/{Techem,Ista,Sonstige,Brunata,Kalo,BFW,Minol}}/merged.VAL-PRED_runtime.coco.json --split 0 --dataset-merged all.VAL-PRED_runtime.coco.json --sort
maskrcnn-formdata merge --dataset {alt.mdl/{Techem,Ista,Sonstige,Brunata},neu.mdl/{Techem,Ista,Sonstige,Brunata,Kalo,BFW,Minol}}/merged.VAL-PRED_runtime.coco.json --split 0 --dataset-merged all.VAL-PRED_runtime.coco.res.json --sort --anns-only
This diff is collapsed.
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment