Pedro Amorim
2951edc697
This commit is a squash of the following: SUSHI harvesting process in the data providers class: * Builds the URL query and requests the SUSHI service endpoint * Parses the JSON response and builds the csv COUNTER file and adds it to counter_files table Usage statistics data processing: * When a counter_files entry is stored, CounterFile.pm will: * Parse the csv COUNTER file and * Add a usage_titles entry for each unique title in the COUNTER file * Add the title's respective erm_usage_mus (monthly usage) entries, repeating for each metric_type * Add the title's respective erm_usage_yus (yearly usage) entries, repeating for each metric_type Harvesting cronjob; 'Run now': * API endpoint to start the harvesting process of a data provider * Button in the data providers list to run the harvesting process for each data provider upon clicked ERM SUSHI: Background job Job progress is updated to total amount of usage titles after retrieving the response from SUSHI; Job warning and success messages are added accordingly Redundant duplicate titles will not be added Redundant duplicate monthly and yearly usage statistics will not be added Data provider harvest background job harvests once per report_type Enqueue one background job for each report_type in the usage data provider Update the way we measure progress in the background job. It now uses the COUNTER report body rows instead of SUSHI response results. We're now incrementing and showing the number of skipped mus, skipped yus, added mus and added yus There's a bug in the way we calculate yus Updates to background job progress bar - Depends on 34468 Signed-off-by: Jessica Zairo <jzairo@bywatersolutions.com> Signed-off-by: Michaela Sieber <michaela.sieber@kit.edu> Signed-off-by: Nick Clemens <nick@bywatersolutions.com> Signed-off-by: Tomas Cohen Arazi <tomascohen@theke.io>
362 lines
13 KiB
Text
362 lines
13 KiB
Text
[% USE raw %]
|
|
[% USE KohaDates %]
|
|
[% USE Asset %]
|
|
[% USE KohaDates %]
|
|
[% PROCESS 'i18n.inc' %]
|
|
[% SET footerjs = 1 %]
|
|
[% INCLUDE 'doc-head-open.inc' %]
|
|
<title>[% FILTER collapse %]
|
|
[% IF op == 'view' %]
|
|
[% tx("Details of job #{job_id}", { job_id = job.id }) | html %] ›
|
|
[% END %]
|
|
[% t("Jobs") | html %] ›
|
|
[% t("Administration") | html %] ›
|
|
[% t("Koha") | html %]
|
|
[% END %]</title>
|
|
|
|
[% INCLUDE 'doc-head-close.inc' %]
|
|
</head>
|
|
|
|
<body id="admin_background_jobs" class="admin">
|
|
[% WRAPPER 'header.inc' %]
|
|
[% INCLUDE 'prefs-admin-search.inc' %]
|
|
[% END %]
|
|
|
|
[% WRAPPER 'sub-header.inc' %]
|
|
[% WRAPPER breadcrumbs %]
|
|
[% IF CAN_user_parameters_manage_background_jobs %]
|
|
[% WRAPPER breadcrumb_item %]
|
|
<a href="/cgi-bin/koha/admin/admin-home.pl">Administration</a>
|
|
[% END %]
|
|
[% IF op == 'view' %]
|
|
[% WRAPPER breadcrumb_item %]
|
|
<a href="/cgi-bin/koha/admin/background_jobs.pl">Jobs</a>
|
|
[% END %]
|
|
[% WRAPPER breadcrumb_item bc_active= 1 %]
|
|
<span>Details of job #[% job.id | html %]</span>
|
|
[% END %]
|
|
[% ELSE %]
|
|
[% WRAPPER breadcrumb_item bc_active= 1 %]
|
|
<span>Jobs</span>
|
|
[% END %]
|
|
[% END %]
|
|
[% ELSE %]
|
|
[% WRAPPER breadcrumb_item bc_active= 1 %]
|
|
<span>Administration</span>
|
|
[% END %]
|
|
[% END %]
|
|
[% END #/ WRAPPER breadcrumbs %]
|
|
[% END #/ WRAPPER sub-header.inc %]
|
|
|
|
<div class="main container-fluid">
|
|
<div class="row">
|
|
<div class="col-sm-10 col-sm-push-2">
|
|
<main>
|
|
|
|
[% FOR m IN messages %]
|
|
<div class="dialog message">
|
|
[% SWITCH m.code %]
|
|
[% CASE 'cannot_view_job' %]
|
|
<div><i class="fa fa-exclamation error"></i>Insufficient permission to see this job.</div>
|
|
[% CASE %]
|
|
[% m.code | html %]
|
|
[% END %]
|
|
</div>
|
|
[% END %]
|
|
|
|
[% IF op == 'view' %]
|
|
<h1>Details of job #[% job.id | html %]</h1>
|
|
|
|
[% PROCESS "background_jobs/${job.type}.inc" %]
|
|
|
|
<div id="job_details" style="display:none">
|
|
<fieldset class="rows">
|
|
<ol>
|
|
<li><span class="label">Job ID: </span>[% job.id | html %]</li>
|
|
<li>
|
|
<label for="job_status">Status: </label>
|
|
<span id="job_status_description"></span>
|
|
</li>
|
|
<li><label for="job_progress">Progress: </label>[% job.progress || 0 | html %] / [% job.size | html %]</li>
|
|
<li>
|
|
<label for="job_type">Type: </label>
|
|
<span id="job_type_description"></span>
|
|
</li>
|
|
<li>
|
|
<label for="job_enqueued_on">Queued: </label>
|
|
[% job.enqueued_on | $KohaDates with_hours = 1 %]
|
|
</li>
|
|
<li>
|
|
<label for="job_started_on">Started: </label>
|
|
[% job.started_on | $KohaDates with_hours = 1 %]
|
|
</li>
|
|
<li>
|
|
<label for="job_ended_on">Ended: </label>
|
|
[% job.ended_on | $KohaDates with_hours = 1 %]
|
|
</li>
|
|
</ol>
|
|
</fieldset>
|
|
<div class="page-section">
|
|
<h2>Report</h2>
|
|
[% IF job.status != 'new' %][% PROCESS 'report' %][% END %]
|
|
</div>
|
|
<div class="page-section">
|
|
<h2>Detailed messages</h2>
|
|
[% IF job.status != 'new' %][% PROCESS 'detail' %][% END %]
|
|
</div>
|
|
</div> <!-- /#job_details -->
|
|
|
|
[% IF CAN_user_parameters_manage_background_jobs %]
|
|
<fieldset class="action">
|
|
<a href="/cgi-bin/koha/admin/background_jobs.pl">Return to the job list</a>
|
|
</fieldset>
|
|
[% END %]
|
|
[% END %]
|
|
|
|
[% IF op == 'list' %]
|
|
|
|
<h1>Jobs</h1>
|
|
|
|
<div>
|
|
<input type="checkbox" id="only_current" checked />
|
|
<label for="only_current">Current jobs only</label>
|
|
</div>
|
|
|
|
<div>
|
|
<input type="checkbox" id="include_last_hour" checked />
|
|
<label for="include_last_hour">Only include jobs enqueued in the last hour</label>
|
|
</div>
|
|
|
|
<div class="page-section">
|
|
<table id="table_jobs">
|
|
<thead>
|
|
<tr>
|
|
<th>Job ID</th>
|
|
<th data-filter="job_statuses">Status</th>
|
|
<th>Progress</th>
|
|
<th data-filter="job_types">Type</th>
|
|
<th>Queued</th>
|
|
<th>Started</th>
|
|
<th>Ended</th>
|
|
<th class="noExport">Actions</th>
|
|
</tr>
|
|
</thead>
|
|
</table>
|
|
</div>
|
|
[% END %]
|
|
|
|
</main>
|
|
</div> <!-- /.col-sm-10.col-sm-push-2 -->
|
|
|
|
<div class="col-sm-2 col-sm-pull-10">
|
|
<aside>
|
|
[% INCLUDE 'admin-menu.inc' %]
|
|
</aside>
|
|
</div> <!-- /.col-sm-2.col-sm-pull-10 -->
|
|
</div> <!-- /.row -->
|
|
|
|
[% MACRO jsinclude BLOCK %]
|
|
[% Asset.js("js/admin-menu.js") | $raw %]
|
|
[% INCLUDE 'js-date-format.inc' %]
|
|
[% INCLUDE 'datatables.inc' %]
|
|
<script>
|
|
const job_statuses = [
|
|
{'_id': 'new', '_str': _("New")},
|
|
{'_id': 'cancelled', '_str': _("Cancelled")},
|
|
{'_id': 'finished', '_str': _("Finished")},
|
|
{'_id': 'started', '_str': _("Started")},
|
|
{'_id': 'running', '_str': _("Running")},
|
|
{'_id': 'failed', '_str': _("Failed")},
|
|
];
|
|
function get_job_status (status) {
|
|
let status_lib = job_statuses.find( s => s._id == status );
|
|
if (status_lib) {
|
|
return status_lib._str;
|
|
}
|
|
return status;
|
|
}
|
|
|
|
const job_types = [
|
|
{
|
|
'_id': 'batch_biblio_record_modification',
|
|
'_str': _("Batch bibliographic record modification")
|
|
},
|
|
{
|
|
'_id': 'batch_biblio_record_deletion',
|
|
'_str': _("Batch bibliographic record deletion")
|
|
},
|
|
{
|
|
'_id': 'batch_authority_record_modification',
|
|
'_str': _("Batch authority record modification")
|
|
},
|
|
{
|
|
'_id': 'batch_authority_record_deletion',
|
|
'_str': _("Batch authority record deletion")
|
|
},
|
|
{
|
|
'_id': 'batch_item_record_modification',
|
|
'_str': _("Batch item record modification")
|
|
},
|
|
{
|
|
'_id': 'batch_item_record_deletion',
|
|
'_str': _("Batch item record deletion")
|
|
},
|
|
{
|
|
'_id': 'erm_sushi_harvester',
|
|
'_str': _("ERM Usage Statistics SUSHI Harvester")
|
|
},
|
|
{
|
|
'_id': 'batch_hold_cancel',
|
|
'_str': _("Batch hold cancellation")
|
|
},
|
|
{
|
|
'_id': 'create_eholdings_from_biblios',
|
|
'_str': _("Create eHolding titles")
|
|
},
|
|
{
|
|
'_id': 'update_elastic_index',
|
|
'_str': _("Update Elasticsearch index")
|
|
},
|
|
{
|
|
'_id': 'update_holds_queue_for_biblios',
|
|
'_str': _("Holds queue update")
|
|
},
|
|
{
|
|
'_id': 'stage_marc_for_import',
|
|
'_str': _("Staged MARC records for import")
|
|
},
|
|
{
|
|
'_id': 'marc_import_commit_batch',
|
|
'_str': _("Import MARC records")
|
|
},
|
|
{
|
|
'_id': 'marc_import_revert_batch',
|
|
'_str': _("Revert import MARC records")
|
|
},
|
|
];
|
|
|
|
function get_job_type (job_type) {
|
|
let job_type_lib = job_types.find( t => t._id == job_type );
|
|
if ( job_type_lib ) {
|
|
return job_type_lib._str;
|
|
}
|
|
return _("Unknown job type '%s'").format(job_type);
|
|
}
|
|
|
|
$(document).ready(function() {
|
|
[% IF op == 'view' %]
|
|
$("#job_status_description").html( get_job_status("[% job.status | html %]") );
|
|
$("#job_type_description").html( get_job_type("[% job.type | html %]") );
|
|
$("#job_details").show();
|
|
[% END %]
|
|
|
|
let additional_filters = {
|
|
enqueued_on: function(){
|
|
let now = new Date();
|
|
if ( $("#include_last_hour").is(":checked") ) {
|
|
now.setHours(now.getHours() - 1);
|
|
return { ">": now.toISOString() };
|
|
} else {
|
|
return { "<": now.toISOString() };
|
|
}
|
|
}
|
|
};
|
|
|
|
let only_current_filter = function(){
|
|
if ( $("#only_current").is(":checked") ) {
|
|
return 'only_current=1';
|
|
} else {
|
|
return 'only_current=0';
|
|
}
|
|
}
|
|
|
|
let jobs_table = $("#table_jobs").kohaTable({
|
|
"ajax": {
|
|
"url": "/api/v1/jobs?" + only_current_filter()
|
|
},
|
|
"order": [[ 1, "desc" ]],
|
|
"columns": [
|
|
{
|
|
"data": "job_id",
|
|
"searchable": true,
|
|
"orderable": true
|
|
},
|
|
{
|
|
"data": "status",
|
|
"searchable": true,
|
|
"orderable": true,
|
|
"render": function(data, type, row, meta) {
|
|
return get_job_status(row.status).escapeHtml();
|
|
}
|
|
},
|
|
{
|
|
"data": "progress,size",
|
|
"searchable": false,
|
|
"orderable": true,
|
|
"render": function(data, type, row, meta) {
|
|
return "%s/%s".format(row.progress, row.size).escapeHtml();
|
|
}
|
|
},
|
|
{
|
|
"data": "type",
|
|
"searchable": true,
|
|
"orderable": true,
|
|
"render": function(data, type, row, meta) {
|
|
return get_job_type(row.type).escapeHtml();
|
|
}
|
|
},
|
|
{
|
|
"data": "enqueued_date",
|
|
"searchable": true,
|
|
"orderable": true,
|
|
"render": function(data, type, row, meta) {
|
|
return $datetime(row.enqueued_date);
|
|
}
|
|
},
|
|
{
|
|
"data": "started_date",
|
|
"searchable": true,
|
|
"orderable": true,
|
|
"render": function(data, type, row, meta) {
|
|
return $datetime(row.started_date);
|
|
}
|
|
},
|
|
{
|
|
"data": "ended_date",
|
|
"searchable": true,
|
|
"orderable": true,
|
|
"render": function(data, type, row, meta) {
|
|
return $datetime(row.ended_date);
|
|
}
|
|
},
|
|
{
|
|
"data": function( row, type, val, meta ) {
|
|
var result = '<a class="btn btn-default btn-xs" role="button" href="/cgi-bin/koha/admin/background_jobs.pl?op=view&id='+ encodeURIComponent(row.job_id) +'"><i class="fa-solid fa-eye aria-hidden="true"></i> '+_("View")+'</a>'+"\n";
|
|
if ( row.status == 'new' || row.status == 'started' ) {
|
|
result += '<a class="btn btn-default btn-xs" role="button" href="/cgi-bin/koha/admin/background_jobs.pl?op=cancel&id='+ encodeURIComponent(row.job_id) +'"><i class="fa fa-trash-can" aria-hidden="true"></i> '+_("Cancel")+'</a>';
|
|
}
|
|
return result;
|
|
},
|
|
"searchable": false,
|
|
"orderable": false
|
|
}
|
|
]
|
|
}, null, 1, additional_filters);
|
|
|
|
$("#include_last_hour").on("change", function(){
|
|
jobs_table.DataTable().draw();
|
|
return false;
|
|
});
|
|
|
|
$("#only_current").on("change", function(){
|
|
jobs_table.DataTable().ajax.url("/api/v1/jobs?" + only_current_filter()).load();
|
|
return false;
|
|
});
|
|
});
|
|
</script>
|
|
[% IF op == 'view' %]
|
|
[% PROCESS 'js' %]
|
|
[% END %]
|
|
[% END %]
|
|
|
|
[% INCLUDE 'intranet-bottom.inc' %]
|