diff --git a/cypress/integration/deposit-admin.spec.js b/cypress/integration/deposit-admin.spec.js
index 87284a20..13bad497 100644
--- a/cypress/integration/deposit-admin.spec.js
+++ b/cypress/integration/deposit-admin.spec.js
@@ -1,300 +1,155 @@
/**
* Copyright (C) 2020 The Software Heritage developers
* See the AUTHORS file at the top-level directory of this distribution
* License: GNU Affero General Public License version 3, or any later version
* See top-level LICENSE file for more information
*/
// data to use as request query response
let responseDeposits;
let expectedOrigins;
describe('Test admin deposit page', function() {
beforeEach(() => {
responseDeposits = [
{
'id': 614,
'external_id': 'ch-de-1',
'reception_date': '2020-05-18T13:48:27Z',
'status': 'done',
'status_detail': null,
'swh_id': 'swh:1:dir:ef04a768',
'swh_id_context': 'swh:1:dir:ef04a768;origin=https://w.s.o/c-d-1;visit=swh:1:snp:b234be1e;anchor=swh:1:rev:d24a75c9;path=/'
},
{
'id': 613,
'external_id': 'ch-de-2',
'reception_date': '2020-05-18T11:20:16Z',
'status': 'done',
'status_detail': null,
'swh_id': 'swh:1:dir:181417fb',
'swh_id_context': 'swh:1:dir:181417fb;origin=https://w.s.o/c-d-2;visit=swh:1:snp:8c32a2ef;anchor=swh:1:rev:3d1eba04;path=/'
},
{
'id': 612,
'external_id': 'ch-de-3',
'reception_date': '2020-05-18T11:20:16Z',
'status': 'rejected',
'status_detail': 'incomplete deposit!',
'swh_id': null,
'swh_id_context': null
}
];
// those are computed from the
expectedOrigins = {
614: 'https://w.s.o/c-d-1',
613: 'https://w.s.o/c-d-2',
612: ''
};
});
- it('Should filter out deposits matching excluding pattern from display', function() {
- cy.adminLogin();
- cy.visit(this.Urls.admin_deposit());
-
- cy.server();
-
- // entry supposed to be excluded from the display by default
- let extraDeposit = {
- 'id': 10,
- 'external_id': 'check-deposit-3',
- 'reception_date': '2020-05-18T11:20:16Z',
- 'status': 'done',
- 'status_detail': null,
- 'swh_id': 'swh:1:dir:fb234417',
- 'swh_id_context': 'swh:1:dir:fb234417;origin=https://w.s.o/c-d-3;visit=swh:1:snp:181417fb;anchor=swh:1:rev:3d166604;path=/'
- };
-
- // of course, that's how to copy a list (an "array")
- let testDeposits = responseDeposits.slice();
- // and add a new element to that array by mutating it...
- testDeposits.push(extraDeposit);
- expectedOrigins[10] = 'https://w.s.o/c-d-3';
-
- // ensure we don't touch the original reference
- expect(responseDeposits.length).to.be.equal(3);
- expect(testDeposits.length).to.be.equal(4);
-
- cy.route({
- method: 'GET',
- url: `${this.Urls.admin_deposit_list()}**`,
- response: {
- 'draw': 10,
- 'recordsTotal': testDeposits.length,
- 'recordsFiltered': testDeposits.length,
- 'data': testDeposits
- }
- }).as('listDeposits');
-
- cy.location('pathname')
- .should('be.equal', this.Urls.admin_deposit());
- cy.url().should('include', '/admin/deposit');
-
- cy.get('#swh-admin-deposit-list')
- .should('exist');
-
- cy.wait('@listDeposits').then((xhr) => {
- let deposits = xhr.response.body.data;
- expect(deposits.length).to.equal(testDeposits.length);
-
- cy.get('#swh-admin-deposit-list').find('tbody > tr').as('rows');
-
- // only 2 entries
- cy.get('@rows').each((row, idx, collection) => {
- let deposit = deposits[idx];
- let responseDeposit = testDeposits[idx];
- cy.log('deposit', deposit);
- cy.log('responseDeposit', responseDeposit);
- expect(deposit.id).to.be.equal(responseDeposit['id']);
- expect(deposit.external_id).to.be.equal(responseDeposit['external_id']);
- expect(deposit.status).to.be.equal(responseDeposit['status']);
- expect(deposit.status_detail).to.be.equal(responseDeposit['status_detail']);
- expect(deposit.swh_id).to.be.equal(responseDeposit['swh_id']);
- expect(deposit.swh_id_context).to.be.equal(responseDeposit['swh_id_context']);
-
- let expectedOrigin = expectedOrigins[deposit.id];
-
- // part of the data, but it should not be displayed (got filtered out)
- if (deposit.external_id === 'check-deposit-3') {
- cy.contains(deposit.status).should('not.be.visible');
- cy.contains(deposit.status_detail).should('not.be.visible');
- cy.contains(deposit.external_id).should('not.be.visible');
- cy.contains(expectedOrigin).should('not.be.visible');
- cy.contains(deposit.swh_id).should('not.be.visible');
- cy.contains(deposit.swh_id_context).should('not.be.visible');
- } else {
- expect(deposit.external_id).to.be.not.equal('check-deposit-3');
- cy.contains(deposit.id).should('be.visible');
- if (deposit.status !== 'rejected') {
- cy.contains(deposit.external_id).should('not.be.visible');
- cy.contains(expectedOrigin).should('be.visible');
- // ensure it's in the dom
- }
- cy.contains(deposit.status).should('be.visible');
- // those are hidden by default, so now visible
- if (deposit.status_detail !== null) {
- cy.contains(deposit.status_detail).should('not.be.visible');
- }
-
- // those are hidden by default
- if (deposit.swh_id !== null) {
- cy.contains(deposit.swh_id).should('not.be.visible');
- cy.contains(deposit.swh_id_context).should('not.be.visible');
- }
- }
- });
-
- // toggling all links and ensure, the previous checks are inverted
- cy.get('a.toggle-col').click({'multiple': true}).then(() => {
- cy.get('#swh-admin-deposit-list').find('tbody > tr').as('rows');
-
- cy.get('@rows').should('have.length', 3);
-
- cy.get('@rows').each((row, idx, collection) => {
- let deposit = deposits[idx];
- let expectedOrigin = expectedOrigins[deposit.id];
-
- // filtered out deposit
- if (deposit.external_id === 'check-deposit-3') {
- cy.contains(deposit.status).should('not.be.visible');
- cy.contains(deposit.status_detail).should('not.be.visible');
- cy.contains(deposit.external_id).should('not.be.visible');
- cy.contains(expectedOrigin).should('not.be.visible');
- cy.contains(deposit.swh_id).should('not.be.visible');
- cy.contains(deposit.swh_id_context).should('not.be.visible');
- } else {
- expect(deposit.external_id).to.be.not.equal('check-deposit-3');
- // ensure it's in the dom
- cy.contains(deposit.id).should('not.be.visible');
- if (deposit.status !== 'rejected') {
- cy.contains(deposit.external_id).should('not.be.visible');
- expect(row).to.contain(expectedOrigin);
- }
-
- expect(row).to.not.contain(deposit.status);
- // those are hidden by default, so now visible
- if (deposit.status_detail !== null) {
- cy.contains(deposit.status_detail).should('be.visible');
- }
-
- // those are hidden by default, so now they should be visible
- if (deposit.swh_id !== null) {
- cy.contains(deposit.swh_id).should('be.visible');
- cy.contains(deposit.swh_id_context).should('be.visible');
- }
- }
- });
- });
-
- cy.get('#swh-admin-deposit-list-error')
- .should('not.contain',
- 'An error occurred while retrieving the list of deposits');
- });
-
- });
-
it('Should display properly entries', function() {
cy.adminLogin();
cy.visit(this.Urls.admin_deposit());
let testDeposits = responseDeposits;
cy.server();
cy.route({
method: 'GET',
url: `${this.Urls.admin_deposit_list()}**`,
response: {
'draw': 10,
'recordsTotal': testDeposits.length,
'recordsFiltered': testDeposits.length,
'data': testDeposits
}
}).as('listDeposits');
cy.location('pathname')
.should('be.equal', this.Urls.admin_deposit());
cy.url().should('include', '/admin/deposit');
cy.get('#swh-admin-deposit-list')
.should('exist');
cy.wait('@listDeposits').then((xhr) => {
cy.log('response:', xhr.response);
cy.log(xhr.response.body);
let deposits = xhr.response.body.data;
cy.log('Deposits: ', deposits);
expect(deposits.length).to.equal(testDeposits.length);
cy.get('#swh-admin-deposit-list').find('tbody > tr').as('rows');
// only 2 entries
cy.get('@rows').each((row, idx, collection) => {
let deposit = deposits[idx];
let responseDeposit = testDeposits[idx];
assert.isNotNull(deposit);
assert.isNotNull(responseDeposit);
expect(deposit.id).to.be.equal(responseDeposit['id']);
expect(deposit.external_id).to.be.equal(responseDeposit['external_id']);
expect(deposit.status).to.be.equal(responseDeposit['status']);
expect(deposit.status_detail).to.be.equal(responseDeposit['status_detail']);
expect(deposit.swh_id).to.be.equal(responseDeposit['swh_id']);
expect(deposit.swh_id_context).to.be.equal(responseDeposit['swh_id_context']);
let expectedOrigin = expectedOrigins[deposit.id];
// ensure it's in the dom
cy.contains(deposit.id).should('be.visible');
if (deposit.status !== 'rejected') {
expect(row).to.not.contain(deposit.external_id);
cy.contains(expectedOrigin).should('be.visible');
}
cy.contains(deposit.status).should('be.visible');
// those are hidden by default, so now visible
if (deposit.status_detail !== null) {
cy.contains(deposit.status_detail).should('not.be.visible');
}
// those are hidden by default
if (deposit.swh_id !== null) {
cy.contains(deposit.swh_id).should('not.be.visible');
cy.contains(deposit.swh_id_context).should('not.be.visible');
}
});
// toggling all links and ensure, the previous checks are inverted
cy.get('a.toggle-col').click({'multiple': true}).then(() => {
cy.get('#swh-admin-deposit-list').find('tbody > tr').as('rows');
cy.get('@rows').each((row, idx, collection) => {
let deposit = deposits[idx];
let expectedOrigin = expectedOrigins[deposit.id];
// ensure it's in the dom
cy.contains(deposit.id).should('not.be.visible');
if (deposit.status !== 'rejected') {
expect(row).to.not.contain(deposit.external_id);
expect(row).to.contain(expectedOrigin);
}
expect(row).to.not.contain(deposit.status);
// those are hidden by default, so now visible
if (deposit.status_detail !== null) {
cy.contains(deposit.status_detail).should('be.visible');
}
// those are hidden by default, so now they should be visible
if (deposit.swh_id !== null) {
cy.contains(deposit.swh_id).should('be.visible');
cy.contains(deposit.swh_id_context).should('be.visible');
}
});
});
cy.get('#swh-admin-deposit-list-error')
.should('not.contain',
'An error occurred while retrieving the list of deposits');
});
});
});
diff --git a/swh/web/admin/deposit.py b/swh/web/admin/deposit.py
index ce48295f..a7e2eb89 100644
--- a/swh/web/admin/deposit.py
+++ b/swh/web/admin/deposit.py
@@ -1,101 +1,111 @@
# Copyright (C) 2018-2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import json
import requests
from django.core.cache import cache
from django.conf import settings
from django.contrib.admin.views.decorators import staff_member_required
from django.core.paginator import Paginator
from django.http import HttpResponse
from django.shortcuts import render
from requests.auth import HTTPBasicAuth
import sentry_sdk
from swh.web.admin.adminurls import admin_route
from swh.web.config import get_config
config = get_config()["deposit"]
@admin_route(r"deposit/", view_name="admin-deposit")
@staff_member_required(view_func=None, login_url=settings.LOGIN_URL)
def _admin_origin_save(request):
return render(request, "admin/deposit.html")
@admin_route(r"deposit/list/", view_name="admin-deposit-list")
@staff_member_required(view_func=None, login_url=settings.LOGIN_URL)
def _admin_deposit_list(request):
table_data = {}
table_data["draw"] = int(request.GET["draw"])
deposits_list_url = config["private_api_url"] + "deposits"
deposits_list_auth = HTTPBasicAuth(
config["private_api_user"], config["private_api_password"]
)
-
try:
nb_deposits = requests.get(
"%s?page_size=1" % deposits_list_url, auth=deposits_list_auth, timeout=30
).json()["count"]
deposits_data = cache.get("swh-deposit-list")
if not deposits_data or deposits_data["count"] != nb_deposits:
deposits_data = requests.get(
"%s?page_size=%s" % (deposits_list_url, nb_deposits),
auth=deposits_list_auth,
timeout=30,
).json()
cache.set("swh-deposit-list", deposits_data)
deposits = deposits_data["results"]
search_value = request.GET["search[value]"]
if search_value:
deposits = [
d
for d in deposits
if any(
search_value.lower() in val
for val in [str(v).lower() for v in d.values()]
)
]
+ exclude_pattern = request.GET.get("excludePattern")
+ if exclude_pattern:
+ deposits = [
+ d
+ for d in deposits
+ if all(
+ exclude_pattern.lower() not in val
+ for val in [str(v).lower() for v in d.values()]
+ )
+ ]
+
column_order = request.GET["order[0][column]"]
field_order = request.GET["columns[%s][name]" % column_order]
order_dir = request.GET["order[0][dir]"]
deposits = sorted(deposits, key=lambda d: d[field_order] or "")
if order_dir == "desc":
deposits = list(reversed(deposits))
length = int(request.GET["length"])
page = int(request.GET["start"]) / length + 1
paginator = Paginator(deposits, length)
data = paginator.page(page).object_list
table_data["recordsTotal"] = deposits_data["count"]
table_data["recordsFiltered"] = len(deposits)
table_data["data"] = [
{
"id": d["id"],
"external_id": d["external_id"],
"reception_date": d["reception_date"],
"status": d["status"],
"status_detail": d["status_detail"],
"swh_id": d["swh_id"],
"swh_id_context": d["swh_id_context"],
}
for d in data
]
except Exception as exc:
sentry_sdk.capture_exception(exc)
table_data["error"] = (
"An error occurred while retrieving " "the list of deposits !"
)
return HttpResponse(json.dumps(table_data), content_type="application/json")
diff --git a/swh/web/assets/src/bundles/admin/deposit.js b/swh/web/assets/src/bundles/admin/deposit.js
index 9dc3ace2..b6e457b3 100644
--- a/swh/web/assets/src/bundles/admin/deposit.js
+++ b/swh/web/assets/src/bundles/admin/deposit.js
@@ -1,210 +1,162 @@
/**
* Copyright (C) 2018-2020 The Software Heritage developers
* See the AUTHORS file at the top-level directory of this distribution
* License: GNU Affero General Public License version 3, or any later version
* See top-level LICENSE file for more information
*/
function genSwhLink(data, type) {
if (type === 'display') {
if (data && data.startsWith('swh')) {
let browseUrl = Urls.browse_swh_id(data);
return `${data}`;
}
}
return data;
}
-function filterDataWithExcludePattern(data, excludePattern) {
- /* Return true if the data is to be filtered, false otherwise.
-
- Args:
- data (dict): row dict data
- excludePattern (str): pattern to lookup in data columns
-
- Returns:
- true if the data is to be excluded (because it matches), false otherwise
-
- */
- if (excludePattern === '') {
- return false; // otherwise, everything gets excluded
- }
- for (const key in data) {
- let value = data[key];
- if ((typeof value === 'string' || value instanceof String) &&
- value.search(excludePattern) !== -1) {
- return true; // exclude the data from filtering
- }
- }
- return false;
-}
-
export function initDepositAdmin() {
let depositsTable;
$(document).ready(() => {
$.fn.dataTable.ext.errMode = 'none';
depositsTable = $('#swh-admin-deposit-list')
.on('error.dt', (e, settings, techNote, message) => {
$('#swh-admin-deposit-list-error').text(message);
})
.DataTable({
serverSide: true,
processing: true,
// let's define the order of table options display
// f: (f)ilter
// l: (l)ength changing
// r: p(r)ocessing
// t: (t)able
// i: (i)nfo
// p: (p)agination
// see https://datatables.net/examples/basic_init/dom.html
- dom: '<l>rt<"bottom"ip>>',
+ dom: '<<"d-flex justify-content-between align-items-center"f' +
+ '<"#list-exclude">l>rt<"bottom"ip>>',
// div#list-exclude is a custom filter added next to dataTable
// initialization below through js dom manipulation, see
// https://datatables.net/examples/advanced_init/dom_toolbar.html
ajax: {
url: Urls.admin_deposit_list(),
- // filtering data set depending on the exclude search input
- dataFilter: function(dataResponse) {
- /* Filter out data returned by the server to exclude entries
- matching the exclude pattern.
-
- Args
- dataResponse (str): the json response in string
-
- Returns:
- json response altered (in string)
- */
- //
- let data = jQuery.parseJSON(dataResponse);
- let excludePattern = $('#swh-admin-deposit-list-exclude-filter').val();
- let recordsFiltered = 0;
- let filteredData = [];
- for (const row of data.data) {
- if (filterDataWithExcludePattern(row, excludePattern)) {
- recordsFiltered += 1;
- } else {
- filteredData.push(row);
- }
- }
- // update data values
- data['recordsFiltered'] = recordsFiltered;
- data['data'] = filteredData;
- return JSON.stringify(data);
+ data: d => {
+ d.excludePattern = $('#swh-admin-deposit-list-exclude-filter').val();
}
},
columns: [
{
data: 'id',
name: 'id'
},
{
data: 'swh_id_context',
name: 'swh_id_context',
render: (data, type, row) => {
if (data && type === 'display') {
let originPattern = ';origin=';
let originPatternIdx = data.indexOf(originPattern);
if (originPatternIdx !== -1) {
let originUrl = data.slice(originPatternIdx + originPattern.length);
let nextSepPattern = ';';
let nextSepPatternIdx = originUrl.indexOf(nextSepPattern);
if (nextSepPatternIdx !== -1) { /* Remove extra context */
originUrl = originUrl.slice(0, nextSepPatternIdx);
}
return `${originUrl}`;
}
}
return data;
}
},
{
data: 'reception_date',
name: 'reception_date',
render: (data, type, row) => {
if (type === 'display') {
let date = new Date(data);
return date.toLocaleString();
}
return data;
}
},
{
data: 'status',
name: 'status'
},
{
data: 'status_detail',
name: 'status_detail',
render: (data, type, row) => {
if (type === 'display' && data) {
let text = data;
if (typeof data === 'object') {
text = JSON.stringify(data, null, 4);
}
return `${text}
`;
}
return data;
},
orderable: false,
visible: false
},
{
data: 'swh_id',
name: 'swh_id',
render: (data, type, row) => {
return genSwhLink(data, type);
},
orderable: false,
visible: false
},
{
data: 'swh_id_context',
name: 'swh_id_context',
render: (data, type, row) => {
return genSwhLink(data, type);
},
orderable: false,
visible: false
}
],
scrollX: true,
scrollY: '50vh',
scrollCollapse: true,
order: [[0, 'desc']]
});
// Some more customization is needed on the table
$('div#list-exclude').html(`
`);
// Adding exclusion pattern update behavior, when typing, update search
$('#swh-admin-deposit-list-exclude-filter').keyup(function() {
depositsTable.draw();
});
// at last draw the table
depositsTable.draw();
});
$('a.toggle-col').on('click', function(e) {
e.preventDefault();
var column = depositsTable.column($(this).attr('data-column'));
column.visible(!column.visible());
if (column.visible()) {
$(this).removeClass('col-hidden');
} else {
$(this).addClass('col-hidden');
}
});
}