Commit 55f69243 authored by vehnem's avatar vehnem
Browse files

mockup: logs, downloads

parent 57ab85aa
......@@ -8,6 +8,7 @@ class ScalatraBootstrap extends LifeCycle {
implicit val swagger = new DataApiSwagger
override def init(context: ServletContext) {
context.setInitParameter("org.scalatra.cors.allowCredentials","false")
context.mount(new DataApiServlet, "/api/*", "data-api")
context.mount(new StaticHtmlServlet, "/*", "static-html")
context.mount(new ResourcesApp, "/api-docs")
......
......@@ -65,26 +65,26 @@ object Config {
|SELECT ?expected_files ?actual_files ?delta ?artifact {
| {SELECT ?expected_files (COUNT(DISTINCT ?distribution) as ?actual_files) ((?actual_files-?expected_files)AS ?delta) ?artifact {
| VALUES (?artifact ?expected_files) {
|( <https://databus.dbpedia.org/dbpedia/generic/anchor-text> 1 )
|( <https://databus.dbpedia.org/dbpedia/generic/article-templates> 278 )
|( <https://databus.dbpedia.org/dbpedia/generic/categories> 417 )
|( <https://databus.dbpedia.org/dbpedia/generic/citations> 2 )
|( <https://databus.dbpedia.org/dbpedia/generic/commons-sameas-links> 7 )
|( <https://databus.dbpedia.org/dbpedia/generic/disambiguations> 15 )
|( <https://databus.dbpedia.org/dbpedia/generic/external-links> 139 )
|( <https://databus.dbpedia.org/dbpedia/generic/geo-coordinates> 139 )
|( <https://databus.dbpedia.org/dbpedia/generic/homepages> 13 )
|( <https://databus.dbpedia.org/dbpedia/generic/infobox-properties> 139 )
|( <https://databus.dbpedia.org/dbpedia/generic/infobox-property-definitions> 139 )
|( <https://databus.dbpedia.org/dbpedia/generic/interlanguage-links> 139 )
|( <https://databus.dbpedia.org/dbpedia/generic/labels> 139 )
|( <https://databus.dbpedia.org/dbpedia/generic/page> 278 )
|( <https://databus.dbpedia.org/dbpedia/generic/persondata> 4 )
|( <https://databus.dbpedia.org/dbpedia/generic/redirects> 139 )
|( <https://databus.dbpedia.org/dbpedia/generic/revisions> 278 )
|( <https://databus.dbpedia.org/dbpedia/generic/topical-concepts>11 )
|( <https://databus.dbpedia.org/dbpedia/generic/wikilinks> 139 )
|( <https://databus.dbpedia.org/dbpedia/generic/wikipedia-links> 139 )
|( <https://databus.dbpedia.org/marvin/generic/anchor-text> 1 )
|( <https://databus.dbpedia.org/marvin/generic/article-templates> 278 )
|( <https://databus.dbpedia.org/marvin/generic/categories> 417 )
|( <https://databus.dbpedia.org/marvin/generic/citations> 2 )
|( <https://databus.dbpedia.org/marvin/generic/commons-sameas-links> 7 )
|( <https://databus.dbpedia.org/marvin/generic/disambiguations> 15 )
|( <https://databus.dbpedia.org/marvin/generic/external-links> 139 )
|( <https://databus.dbpedia.org/marvin/generic/geo-coordinates> 139 )
|( <https://databus.dbpedia.org/marvin/generic/homepages> 13 )
|( <https://databus.dbpedia.org/marvin/generic/infobox-properties> 139 )
|( <https://databus.dbpedia.org/marvin/generic/infobox-property-definitions> 139 )
|( <https://databus.dbpedia.org/marvin/generic/interlanguage-links> 139 )
|( <https://databus.dbpedia.org/marvin/generic/labels> 139 )
|( <https://databus.dbpedia.org/marvin/generic/page> 278 )
|( <https://databus.dbpedia.org/marvin/generic/persondata> 4 )
|( <https://databus.dbpedia.org/marvin/generic/redirects> 139 )
|( <https://databus.dbpedia.org/marvin/generic/revisions> 278 )
|( <https://databus.dbpedia.org/marvin/generic/topical-concepts>11 )
|( <https://databus.dbpedia.org/marvin/generic/wikilinks> 139 )
|( <https://databus.dbpedia.org/marvin/generic/wikipedia-links> 139 )
| }
| ?dataset dataid:artifact ?artifact .
| ?dataset dct:hasVersion ?versionString .
......@@ -136,7 +136,7 @@ object Config {
object extractionLogs {
object baseUrl {
val mappings = new URL("http://dbpedia-mappings.tib.eu/logs/")
val generic = new URL("http://dbpedia-generic.tib.eu/logs/")
val generic = new URL("http://dbpedia-mappings.tib.eu/logs/")
val wikidata = new URL("http://dbpedia-wikidata.tib.eu/logs/")
}
......
......@@ -91,9 +91,10 @@
<div class="col-sm-5 p-3" id="placeholder">
<h2 class="text-info">TODO</h2>
<ul>
<li>rerelease</li>
<li>link to databus</li>
<li>other download chart</li>
<li>logs: data</li>
<li>wikidumps: chart data</li>
<li>links to databus</li>
<li>re-release check</li>
</ul>
</div>
<!-- end of grid -->
......@@ -145,6 +146,17 @@
</div>
</div>
</div>
<div class="col-sm-6 p-3">
<div class="card">
<div class="card-body">
<h4>Triples Chart</h4>
<p>
TODO: file size<br>
TODO: triples per month
</p>
</div>
</div>
</div>
<!-- end of grid-->
</div>
</div>
......@@ -231,7 +243,7 @@
<div class="card-body">
<h4>Wikimedia Dumps</h4>
<div class="row">
<div id="mappings-downloads" class="col-sm-8" style="height: 250px;">
<div id="mappings-downloads-chart" class="col-sm-8" style="height: 250px;">
Dump Status...
</div>
<div class="col-sm-4">
......@@ -248,7 +260,7 @@
<div class="card">
<div class="card-body">
<h4>Log Files</h4>
<table id="mappings-logs" data-pagination="true"
<table id="mappings-logs-table" data-pagination="true"
data-page-size="4" class="table">
<thead>
<th data-field="state">State</th>
......@@ -307,20 +319,33 @@
<div class="col-sm-6 p-3">
<div class="card">
<div class="card-body">
<h4>Medawiki Dump Download Completeness</h4>
<p>
Checks whether the input of MARVIN was downloaded correctly from dumps.wikimedia.org
</p>
<h4>Wikimedia Dumps</h4>
<div class="row">
<div id="generic-downloads-chart" class="col-sm-8" style="height: 250px;">
Dump Status...
</div>
<div class="col-sm-4">
<h6 class="text-danger">Missing files </h6>
<p>
<strong>TODO</strong> List of missing languages
</p>
</div>
</div>
</div>
</div>
</div>
<div class="col-sm-6 p-3">
<div class="card">
<div class="card-body">
<h4>Downloadable Log Files</h4>
<p>
n/a - not loaded, unzipped - in progress, zipped - done (but success unclear)
</p>
<h4>Log Files</h4>
<table id="generic-logs-table" data-pagination="true"
data-page-size="4" class="table">
<thead>
<th data-field="state">State</th>
<th data-field="filename">Log File</th>
<th data-field="description">Description</th>
</thead>
</table>
</div>
</div>
</div>
......@@ -372,20 +397,33 @@
<div class="col-sm-6 p-3">
<div class="card">
<div class="card-body">
<h4>Medawiki Dump Download Completeness</h4>
<p>
Checks whether the input of MARVIN was downloaded correctly from dumps.wikimedia.org
</p>
<h4>Wikimedia Dumps</h4>
<div class="row">
<div id="wikidata-downloads-chart" class="col-sm-8" style="height: 250px;">
Dump Status...
</div>
<div class="col-sm-4">
<h6 class="text-danger">Missing files </h6>
<p>
<strong>TODO</strong> List of missing languages
</p>
</div>
</div>
</div>
</div>
</div>
<div class="col-sm-6 p-3">
<div class="card">
<div class="card-body">
<h4>Downloadable Log Files</h4>
<p>
n/a - not loaded, unzipped - in progress, zipped - done (but success unclear)
</p>
<h4>Log Files</h4>
<table id="wikidata-logs-table" data-pagination="true"
data-page-size="4" class="table">
<thead>
<th data-field="state">State</th>
<th data-field="filename">Log File</th>
<th data-field="description">Description</th>
</thead>
</table>
</div>
</div>
</div>
......
......@@ -2,9 +2,13 @@ const api = "http://localhost:8080/api/"
/* anchor scroll offset */
window.addEventListener("hashchange", function () {
window.scrollTo(window.scrollX, window.scrollY - 80);
window.scrollTo(window.scrollX, window.scrollY - 80);
});
if (window.location.hash) {
window.scrollTo(window.scrollX, window.scrollY - 80);
}
/* versions */
function latestDate() {
......@@ -38,88 +42,84 @@ $.getJSON(api + "release/versions", function (data) {
/* overall */
/* mappings-dumps */
/* dump-downloads */
google.charts.load('current', { 'packages': ['corechart'] });
google.charts.setOnLoadCallback(drawChart);
function drawDumpChart(group, fin, wait, miss) {
google.charts.load('current', { 'packages': ['corechart'] });
google.charts.setOnLoadCallback(drawChart);
function drawChart() {
function drawChart() {
var data = google.visualization.arrayToDataTable([
['State', 'Number'],
['Finished', 32],
['Waiting', 5],
['Mssing', 3]
]);
var data = google.visualization.arrayToDataTable([
['State', 'Number'],
['Finished', fin],
['Waiting', wait],
['Mssing', miss]
]);
var options = {
colors: ['#28a745', '#17a2b8', '#dc3545'],
pieHole: 0.5,
pieSliceText: 'value',
};
var options = {
colors: ['#28a745', '#17a2b8', '#dc3545'],
pieHole: 0.5,
pieSliceText: 'value',
};
var chart = new google.visualization.PieChart(document.getElementById('mappings-downloads'));
chart.draw(data, options);
var chart = new google.visualization.PieChart(document.getElementById(`${group}-downloads-chart`));
chart.draw(data, options);
}
}
/* mappings-logs */
$(function () {
drawDumpChart('mappings', 36, 2, 2)
drawDumpChart('generic', 138, 2, 2)
drawDumpChart('wikidata', 1, 0, 0)
});
// n/a - not loaded,
// unzipped - in progress,
// zipped - done (but success unclear)
var mappingsLogsTable = $('#mappings-logs')
/* log-files */
$(function () {
var data = [
{
'filename': '<a href="#">preprocess.log</a>',
'description': 'preprocessing (todo explain)',
'state': '<span class="text-warning">RUN</span>'
},
{
'filename': '<a href="#">extraction.mappings.log</a>',
'description': 'process of DIEF',
'state': '<span class="text-success">DONE</span>'
},
{
'filename': '<a href="#">dump.log</a>',
'description': 'download of latest dumps',
'state': '<span class="text-success">DONE</span>'
},
{
'filename': '<a href="#">ontology.log</a>',
'description': 'download of latest ontology',
'state': '<span class="text-success">DONE</span>'
},
{
'filename': '<a href="#">mappings.log</a>',
'description': 'download of latest mappings',
'state': '<span class="text-success">DONE</span>'
},
{
'filename': 'deploy.log',
'description': 'deploy on databus',
'state': '<span class="text-danger">WAIT</span>'
}
]
mappingsLogsTable.bootstrapTable({ data: data })
})
function getLogs(group) {
// n/a - not loaded,
// unzipped - in progress,
// zipped - done (but success unclear)
/* mappings-completness */
var logTable = $(`#${group}-logs-table`)
var mappingsCompletenessTable = $('#mappings-completeness')
$.getJSON(api + `release/logs/wikidata/2020.05.01`, function (data) {
$(function () {
$.getJSON(api + `release/completeness/mappings/${version}`, function (data) {
var processLogs = []
data.forEach(element => {
var stateVal = 0
var state = element.state
if (state == 'WAIT') {
stateVal = 0
state = '<strong class="text-warning">WARN</strong>'
} else if (state == 'RUN') {
tateVal = 1
state = '<strong class="text-success">RUN</strong>'
} else {
stateVal = 2
state = '<strong class="text-info">DONE</strong>'
}
var url = element.url
var file = element.logName
var description = 'TODO'
processLogs.push({ 'state': state, 'description': description, 'filename': `<a href="${url}">${file}</a>` })
})
logTable.bootstrapTable({ 'data': processLogs })
});
});
}
/* generic */
/* wikidata-completness */
$(function () {
getLogs('mappings')
getLogs('generic')
getLogs('wikidata')
})
http://localhost:8080/api/release/logs/wikidata/2020.05.01
function checkCompleteness(group, expectedArtifacts) {
......@@ -138,7 +138,6 @@ function checkCompleteness(group, expectedArtifacts) {
expectedFilesTotal += exp
actualFilesTotal += act
var stateVal = act < exp ? 0 : 1
var state = act < exp ? '<strong class="text-warning">WARN</strong>' : '<strong class="text-info">OK</strong>'
var missing = act < exp ? exp - act : 0
......@@ -146,44 +145,46 @@ function checkCompleteness(group, expectedArtifacts) {
'artifact': element['artifact'],
'state': state,
'missing': missing,
'stateVal': stateVal
});
});
releaseCompleteness.sort((a, b) => a.stateVal - b.stateVal)
releaseCompleteness.sort((a, b) => b.missing - a.missing)
mappingsCompletenessTable.bootstrapTable({ 'data': releaseCompleteness })
let progBarArt = $(`#${group}-completeness-artifacts`)
progBarArt.html(`${actualArtifacts}/${expectedArtifacts} `)
let progArt = actualArtifacts * 100 / expectedArtifacts
progBarArt.css('width', progArt + '%')
if (progArt >= 100.0 ) progBarArt.addClass('bg-info')
else if (progArt < 50) progBarArt.addClass('bg-danger')
else progBarArt.addClass('bg-warning')
setProgressBar(progBarArt, actualArtifacts, expectedArtifacts)
// progBarArt.html(`${actualArtifacts}/${expectedArtifacts} `)
// let progArt = actualArtifacts * 100 / expectedArtifacts
// progBarArt.css('width', progArt + '%')
// if (progArt >= 100.0) progBarArt.addClass('bg-info')
// else if (progArt < 50) progBarArt.addClass('bg-danger')
// else progBarArt.addClass('bg-warning')
let progBarFil = $(`#${group}-completeness-files`)
progBarFil.html(`${actualFilesTotal}/${expectedFilesTotal}`)
let progFil = actualFilesTotal * 100 / expectedFilesTotal
progBarFil.css('width', progFil + '%')
if (progFil >= 100) progBarFil.addClass('bg-info')
else if (progFil < 50) progBarFil.addClass('bg-danger')
else progBarFil.addClass('bg-warning')
setProgressBar(progBarFil, actualFilesTotal, expectedFilesTotal)
// progBarFil.html(`${actualFilesTotal}/${expectedFilesTotal}`)
// let progFil = actualFilesTotal * 100 / expectedFilesTotal
// progBarFil.css('width', progFil + '%')
// if (progFil >= 100) progBarFil.addClass('bg-info')
// else if (progFil < 50) progBarFil.addClass('bg-danger')
// else progBarFil.addClass('bg-warning')
});
}
$(function() {
checkCompleteness('mappings',6)
checkCompleteness('generic',20)
checkCompleteness('wikidata',16)
function setProgressBar(bar, actual, max) {
bar.html(`$(${actual}/${expected})`)
var percentage = actual * 100 / max
bar.css('width', percentage + '%')
if (percentage >= 100.0)
bar.addClass('bg-info')
else if (actual == 0.0)
bar.css('width', '100%')
bar.addClass('bg-danger')
else
bar.addClass('bg-warning')
}
// checkCompleteness('mapp')
$(function () {
checkCompleteness('mappings', 6)
checkCompleteness('generic', 20)
checkCompleteness('wikidata', 16)
})
/** FUNCTIONS **/
/* get latest version from api or from date*/
// function getLatestVersion() {
// // TODO
// return "2020.05.01"
// }
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment