Commit a84cac52 authored by vehnem's avatar vehnem
Browse files

api: version, mappings completness

parent 659c2ff9
databus-poms/dbpedia/*/*/*/
extraction-framework/
wikidumps/
logs/
release/
marvin-extraction/
pom.xml.versionsBackup
/databus-poms/dbpedia/*/*/*/
/extraction-framework/
/wikidumps/
/logs/
/release/
/marvin-extraction/
/pom.xml.versionsBackup
package org.dbpedia.release
case class CompletenessResponse
(
artifactIri: String,
versionStr: String,
expectedFiles: String,
actualFiles: String,
)
package org.dbpedia.release
/**
* Dashboard config
*/
object Config {
}
package org.dbpedia.release
import org.apache.jena.query.{QueryException, QueryExecutionFactory}
import org.json4s.{DefaultFormats, Formats}
import org.scalatra._
import org.scalatra.json.JacksonJsonSupport
import org.slf4j.LoggerFactory
import scala.collection.mutable.ArrayBuffer
// JSON-related libraries
import org.json4s.{DefaultFormats, Formats}
// JSON handling support from Scalatra
import org.scalatra.json._
class DataApiServlet extends ScalatraServlet with JacksonJsonSupport {
val log = LoggerFactory.getLogger(classOf[DataApiServlet])
before() {
contentType = formats("json")
}
protected implicit val jsonFormats: Formats = DefaultFormats
/**
*
*/
get("/release/versions/:group") {
val group = params("group")
val marvinVersions = new ArrayBuffer[String]()
val dbpediaVersions = new ArrayBuffer[String]()
QueryExecutionFactory
.sparqlService("http://databus.dbpedia.org/repo/sparql", Queries.versionQuery(group))
.execSelect()
.forEachRemaining(row => {
val version = row.get("?version").asLiteral().getLexicalForm
val account = row.get("account").asResource().getURI
if ("https://databus.dbpedia.org/marvin" == account)
marvinVersions.append(version)
else {
dbpediaVersions.append(version)
}
})
VersionResponse(group, marvinVersions.toList, dbpediaVersions.toList)
}
/**
* source files completeness for given release
*/
get("/check-download/:account/:group/:artifact/:version") {
"TODO"
}
/**
* target files completeness for given release
*/
// get("/check-release/:account/:group/:artifact/:version") {
get("/check-release/:group") {
val group = params("group")
val query = {
group match {
case "mappings" => Queries.complMappings()
case "generic" => ""
case "wikidata" => ""
case "text" => ""
}
}
log.info("\n"+query)
val response = new ArrayBuffer[CompletenessResponse]()
QueryExecutionFactory
.sparqlService("http://databus.dbpedia.org/repo/sparql", query)
.execSelect()
.forEachRemaining(row => {
val artifact = row.get("artifact").asResource().getURI
val version = row.get("versionString").asLiteral().getLexicalForm
val actualFiles = row.get("actual_files").asLiteral().getLexicalForm
val expectedFiles = row.get("expected_files").asLiteral().getLexicalForm
response.append(CompletenessResponse(artifact,version,actualFiles,expectedFiles))
})
response.toList
}
/**
* log state for given release
*/
get("/check-logs/:account/:group/:artifact/:version") {
"TODO"
}
}
package org.dbpedia.release
object Queries {
def versionQuery(group: String): String =
s"""PREFIX dataid: <http://dataid.dbpedia.org/ns/core#>
|PREFIX dct: <http://purl.org/dc/terms/>
|
|SELECT DISTINCT ?account ?group ?version WHERE {
| VALUES ?group {
| <https://databus.dbpedia.org/marvin/$group>
| <https://databus.dbpedia.org/dbpedia/$group>
| }
| ?s a dataid:Dataset .
| ?s dataid:account ?account .
| ?s dataid:group ?group .
| ?s dct:hasVersion ?version .
|}
|ORDER BY DESC(?version)
|""".stripMargin
def complMappings(): String =
s"""PREFIX dataid: <http://dataid.dbpedia.org/ns/core#>
|PREFIX dct: <http://purl.org/dc/terms/>
|PREFIX dcat: <http://www.w3.org/ns/dcat#>
|
|SELECT * {
|{SELECT ?expected_files (COUNT(DISTINCT ?distribution) as ?actual_files) ((?actual_files-?expected_files)AS ?delta) ?versionString ?artifact {
| VALUES (?artifact ?expected_files) {
| ( <https://databus.dbpedia.org/dbpedia/mappings/geo-coordinates-mappingbased> 29 )
| ( <https://databus.dbpedia.org/dbpedia/mappings/instance-types> 80 )
| ( <https://databus.dbpedia.org/dbpedia/mappings/mappingbased-literals> 40 )
| ( <https://databus.dbpedia.org/dbpedia/mappings/mappingbased-objects> 120 )
| ( <https://databus.dbpedia.org/dbpedia/mappings/mappingbased-objects-uncleaned> 40 )
| ( <https://databus.dbpedia.org/dbpedia/mappings/specific-mappingbased-properties> 40 )
| }
| ?dataset dataid:artifact ?artifact .
| ?dataset dataid:version ?versionIRI .
| ?dataset dct:hasVersion ?versionString .
| ?dataset dcat:distribution ?distribution .
|} GROUP BY ?versionString ?artifact ?expected_files ?actual_files}
| FILTER(?delta != 0)
|} ORDER BY DESC(?versionString)
|""".stripMargin
}
package org.dbpedia.release
import java.io.File
import org.scalatra.{InternalServerError, NotFound, ScalatraServlet}
class StaticHtmlServlet extends ScalatraServlet {
/**
* serve static files
*
* @param path relative path to 'http(s)://host:port/' and 'src/main/webapp/'
* @return static file content
*/
def html(path: String): File = {
contentType = "text/html"
new File(getServletContext.getResource(path).getFile)
}
/**
* default page for HttpStatus 404 Not Found
*/
notFound {
serveStaticResource() getOrElse NotFound(html("/notFound.html"))
}
/**
* default page for HttpStatus 500 Internal Server
*/
error {
case e: Exception => InternalServerError(html("/error.html"))
}
}
package org.dbpedia.release
case class VersionResponse
(
group: String,
marvin: List[String],
dbpedia: List[String]
)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment