From 84a7891f8cc4f85ec4c90f79e4bb618cd2bfa7a0 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Thu, 4 Apr 2024 15:30:28 +0300 Subject: [PATCH] refactor --- html/autolist.js | 1130 +++++++++++++++++-------------- html/main.js | 1280 +++++++++++++++++++----------------- src/app_state.rs | 10 +- src/datasource_database.rs | 21 +- src/platform.rs | 32 +- src/wdfist.rs | 513 +++++++++------ 6 files changed, 1639 insertions(+), 1347 deletions(-) diff --git a/html/autolist.js b/html/autolist.js index 4a20f68..aaa6fab 100644 --- a/html/autolist.js +++ b/html/autolist.js @@ -1,351 +1,424 @@ -function WiDaR ( callback ) { - - this.is_logged_in = false ; - this.server = 'https://tools.wmflabs.org' ; - this.api = this.server+'/widar/index.php?callback2=?' ; - this.userinfo = {} ; - this.tool_hashtag = '' ; - - this.isLoggedIn = function () { - return this.is_logged_in ; - } - - this.getInfo = function () { - var me = this ; - this.abstractCall ( { - action:'get_rights' - } , function ( d ) { - me.is_logged_in = false ; - me.userinfo = {} ; - if ( typeof (((d||{}).result||{}).query||{}).userinfo == 'undefined' ) { - callback() ; - return ; - } - me.userinfo = d.result.query.userinfo ; - if ( typeof me.userinfo.name != 'undefined' ) me.is_logged_in = true ; - callback() ; - } ) ; - } - - this.getLoginLink = function ( text ) { - var h = "" + text + "" ; - return h ; - } - - this.isBot = function () { - if ( !this.isLoggedIn() ) return false ; - if ( typeof this.userinfo.groups == 'undefined' ) return false ; - return -1 != $.inArray ( 'bot' , this.userinfo.groups ) ; - } - - this.getUserName = function () { - if ( !this.isLoggedIn() ) return ; - return this.userinfo.name ; - } - - this.abstractCall = function ( params , callback ) { - var me = this ; - params.tool_hashtag = me.tool_hashtag ; - params.botmode = 1 ; - params.server = autolist_wiki_server; - $.getJSON ( me.api , params , function ( d ) { - if ( typeof callback != 'undefined' ) callback ( d ) ; - } ) . fail ( function () { - if ( typeof callback != 'undefined' ) callback () ; - } ) ; - } - - this.genericAction = function ( o , callback ) { - this.abstractCall ( { - action:'generic', - json:JSON.stringify(o) - } , callback ) ; - } - - this.removeClaim = function ( id , callback ) { - this.abstractCall ( { - action:'remove_claim', - id:id - } , callback ) ; - } - - this.createNewItem = function ( page , wiki , from_redlink , callback ) { - var me = this ; - if ( from_redlink ) { - this.abstractCall ( { - action:'create_blank_item', - botmode:1 - } , function ( d ) { - me.abstractCall ( { - action:'set_label', - q:d.q, - lang:wiki.replace(/wik.+$/,'').replace(/commons/,'en'), - label:page - } , function ( d2 ) { - callback ( d ) ; - } ) ; - } ) ; - } else { - this.abstractCall ( { - action:'create_item_from_page', - botmode:1, - site:wiki, - page:page - } , callback ) ; - } - } - - this.setClaim = function ( q , prop , target_q , callback ) { - this.abstractCall ( { - action:'set_claims', - ids:q, - prop:prop, - target:target_q - } , callback ) ; - } - - - this.getInfo() ; +function WiDaR(callback) { + this.is_logged_in = false; + this.server = "https://tools.wmflabs.org"; + this.api = this.server + "/widar/index.php?callback2=?"; + this.userinfo = {}; + this.tool_hashtag = ""; + + this.isLoggedIn = function () { + return this.is_logged_in; + }; + + this.getInfo = function () { + var me = this; + this.abstractCall( + { + action: "get_rights", + }, + function (d) { + me.is_logged_in = false; + me.userinfo = {}; + if ( + typeof (((d || {}).result || {}).query || {}).userinfo == "undefined" + ) { + callback(); + return; + } + me.userinfo = d.result.query.userinfo; + if (typeof me.userinfo.name != "undefined") me.is_logged_in = true; + callback(); + }, + ); + }; + + this.getLoginLink = function (text) { + var h = + "" + + text + + ""; + return h; + }; + + this.isBot = function () { + if (!this.isLoggedIn()) return false; + if (typeof this.userinfo.groups == "undefined") return false; + return -1 != $.inArray("bot", this.userinfo.groups); + }; + + this.getUserName = function () { + if (!this.isLoggedIn()) return; + return this.userinfo.name; + }; + + this.abstractCall = function (params, callback) { + var me = this; + params.tool_hashtag = me.tool_hashtag; + params.botmode = 1; + params.server = autolist_wiki_server; + $.getJSON(me.api, params, function (d) { + if (typeof callback != "undefined") callback(d); + }).fail(function () { + if (typeof callback != "undefined") callback(); + }); + }; + + this.genericAction = function (o, callback) { + this.abstractCall( + { + action: "generic", + json: JSON.stringify(o), + }, + callback, + ); + }; + + this.removeClaim = function (id, callback) { + this.abstractCall( + { + action: "remove_claim", + id: id, + }, + callback, + ); + }; + + this.createNewItem = function (page, wiki, from_redlink, callback) { + var me = this; + if (from_redlink) { + this.abstractCall( + { + action: "create_blank_item", + botmode: 1, + }, + function (d) { + me.abstractCall( + { + action: "set_label", + q: d.q, + lang: wiki.replace(/wik.+$/, "").replace(/commons/, "en"), + label: page, + }, + function (d2) { + callback(d); + }, + ); + }, + ); + } else { + this.abstractCall( + { + action: "create_item_from_page", + botmode: 1, + site: wiki, + page: page, + }, + callback, + ); + } + }; + + this.setClaim = function (q, prop, target_q, callback) { + this.abstractCall( + { + action: "set_claims", + ids: q, + prop: prop, + target: target_q, + }, + callback, + ); + }; + + this.getInfo(); } //________________________________________________________________________________________________________________________________________________ -function AutoList ( callback ) { - - this.running = [] ; - this.max_concurrent = 1 ; // 1 thread for non-bot user - this.concurrent = 1 ; - this.delay = 2000 ; // 2 sec delay for non-bot user - - this.setupCommands = function () { - var me = this ; - me.commands_todo = [] ; - me.running = [] ; - var rows = $('#al_commands').val().split("\n") ; - $('#main_table input.qcb').each ( function () { - var o = $(this) ; - if ( !o.is(':checked') ) return ; - var tr = $(o.parents('tr').get(0)) ; - var q = o.attr('q') ; - var remove_q ; - - if ( me.mode == 'creator' ) { - var cmd = { q:q , status:'waiting' , mode:'create' } ; - var a = $($(tr.find("td").get(2)).find('a').get(0)) ; - if ( a.hasClass('redlink') ) cmd.from_redlink = true ; - cmd.page = decodeURIComponent ( a.attr('href').replace(/^.+?\/wiki\//,'') ) ; - remove_q = me.commands_todo.length ; - me.commands_todo.push ( cmd ) ; - } - - $.each ( rows , function ( k , row ) { - let cmd = { q:q , status:'waiting' } ; - let m = row.match ( /^\s*D(\S+):(.+)$/i ) ; - if ( m != null ) { - cmd.mode = 'desc'; - cmd.language = m[1] ; - cmd.value = m[2].replace(/^"(.+?)"$/,"$1") ; - } else { - m = row.match ( /^\s*-(P\d+)/i ) ; - if ( m == null ) { - m = row.match ( /^\s*(P\d+)\s*:\s*(Q\d+)\s*$/i ) ; - if ( m != null ) { - cmd.mode = 'add' ; - cmd.prop = m[1] ; - cmd.value = m[2] ; - } else return ; - } else { // Delete property - cmd.mode = 'delete' ; - cmd.prop = m[1] ; - m = row.match ( /^\s*-(P\d+)\s*:\s*(Q\d+)/i ) ; - if ( m != null ) cmd.value = m[2] ; - } - } - remove_q = me.commands_todo.length ; - me.commands_todo.push ( cmd ) ; - } ) ; - if ( typeof remove_q != 'undefined' ) { - me.commands_todo[remove_q].remove_q = true ; - me.commands_todo[remove_q].cb_id = o.attr('id') ; - } - } ) ; - } - - this.finishCommand = function ( id ) { - var me = this ; - me.commands_todo[id].status = 'done' ; - var cmd = me.commands_todo[id] ; - - if ( cmd.remove_q ) { // Last for this q, remove row - $($('#'+cmd.cb_id).parents('tr').get(0)).remove() ; - } - - var tmp = [] ; - $.each ( me.running , function ( k , v ) { - if ( v != id ) tmp.push ( v ) ; - } ) ; - me.running = tmp ; - - setTimeout ( function () { me.runNextCommand() } , me.delay ) ; - } - - this.addNewQ = function ( q ) { - if ( $('#autolist_box_new_q').length == 0 ) { - $('#autolist_box').append ( "
" ) ; - tt.updateInterface ( $('#autolist_box') ) ; - } - var t = $('#autolist_box_new_q').val() ; - if ( t != '' ) t += "\n" ; - t += "Q" + q ; - $('#autolist_box_new_q').val(t) ; - } - - this.runCommand = function ( id ) { - var me = this ; - me.running.push ( id ) ; - me.commands_todo[id].status = 'running' ; - var cmd = me.commands_todo[id] ; - - if ( cmd.mode == 'create' ) { - - me.widar.createNewItem ( cmd.page , output_wiki , cmd.from_redlink , function ( d ) { - // TODO error check: d.error=="OK" - if ( typeof d == 'undefined' || d.error != 'OK' ) { - if ( typeof d.error == 'object' ) { - if ( d.error.code == 'no-external-page' ) { - console.log ( cmd.page + " does not exist on " + output_wiki + "; maybe it has been deleted?" ) ; - me.finishCommand ( id ) ; - return ; - } - } - console.log ( d ) ; - } - - // Update all subsequent commands for this item to use the new Q - var new_q = d.q.replace(/\D/,'') ; - var old_q = cmd.q ; - me.addNewQ ( new_q ) ; - $.each ( me.commands_todo , function ( k , v ) { - if ( v.q == old_q ) me.commands_todo[k].q = new_q ; - } ) ; - - // Next - me.finishCommand ( id ) ; - } ) ; - - } else if ( cmd.mode == 'add' ) { - me.widar.setClaim ( 'Q'+cmd.q , cmd.prop , cmd.value , function ( d ) { - // TODO error log - me.finishCommand ( id ) ; - } ) ; - } else if ( cmd.mode == 'delete' ) { - $.getJSON ( 'https://'+autolist_wiki_server+'/w/api.php?action=wbgetentities&ids=Q'+cmd.q+'&format=json&callback=?' , function ( d ) { - var done = false ; - $.each ( ((((d.entities||{})['Q'+cmd.q]||{}).claims||{})[cmd.prop.toUpperCase()]||{}) , function ( k , v ) { - if ( typeof cmd.value != 'undefined' ) { // Specific value to delete - if ( ((((v.mainsnak||{}).datavalue||{}).value||{})['numeric-id']||'') != cmd.value ) return ; - } - done = true ; - me.widar.removeClaim ( v.id , function ( d ) { - // TODO error log - console.log ( d ) ; - me.finishCommand ( id ) ; - } ) ; - return false ; // Remove just the first one - } ) ; - if ( !done ) { - // TODO error log - me.finishCommand ( id ) ; - } - } ) . fail ( function () { - // TODO error log - me.finishCommand ( id ) ; - } ) ; - } else { - console.log ( "Unknown mode: " + cmd.mode ) ; - } - } - - this.allDone = function () { - $('#al_do_stop').hide() ; - $('#al_do_process').show() ; - $('#al_start_qs').show() ; - $('#al_status').html ( ""+_t('al_done')+"" ) ; - } - - this.runNextCommand = function () { - var me = this ; - -//console.log ( me.concurrent , me.running.length ) ; - - if ( me.emergency_stop ) return ; // Used clicked stop - - if ( me.running.length >= me.concurrent ) { - setTimeout ( function () { me.runNextCommand } , 100 ) ; // Was already called, so short delay - return ; - } - - var q_running = {} ; - $.each ( me.running , function ( k , v ) { - q_running[me.commands_todo[v].q] = 1 ; - } ) ; - - var run_next ; - var has_commands_left = false ; - $.each ( me.commands_todo , function ( k , v ) { - if ( v.status != 'done' ) has_commands_left = true ; - if ( v.status != 'waiting' ) return ; // Already running - if ( typeof q_running[v.q] != 'undefined' ) return ; // Don't run the same q multiple times - run_next = k ; - return false ; - } ) ; - - if ( !has_commands_left ) { - me.allDone() ; - return ; - } - - if ( typeof run_next == 'undefined' ) { - setTimeout ( function () { me.runNextCommand } , 100 ) ; // Was already called, so short delay - return ; - } +function AutoList(callback) { + this.running = []; + this.max_concurrent = 1; // 1 thread for non-bot user + this.concurrent = 1; + this.delay = 2000; // 2 sec delay for non-bot user - me.updateRunCounter() ; - me.runCommand ( run_next ) ; - } - - this.updateRunCounter = function () { - var me = this ; - var t = _t('al_running') ; - var all = me.commands_todo.length ; - var cnt = 0 ; - $.each ( me.commands_todo , function ( k , v ) { - if ( v.status != 'done' ) cnt++ ; - } ) ; - t = t.replace ( '$1' , cnt ) ; - t = t.replace ( '$2' , all ) ; - $('#al_status').html ( t ) ; - } - - this.is_wikidata = function () { - return typeof autolist_wiki_server=='undefined' || autolist_wiki_server=='' || autolist_wiki_server=='www.wikidata.org' - } - - this.is_commons = function () { - return autolist_wiki_server == 'commons.wikimedia.org' - } - - this.initializeAutoListBox = function () { - var me = this ; - var h = '' ; - var p = getUrlVars() ; - //if ( me.widar.isLoggedIn() ) { - h += "
" ; - //h += "
" + _t('al_welcome').replace( '$1', me.widar.getUserName() ) + "
" ; - if ( me.mode == "creator" ) { - h += "
" ; - } - /* + this.setupCommands = function () { + var me = this; + me.commands_todo = []; + me.running = []; + var rows = $("#al_commands").val().split("\n"); + $("#main_table input.qcb").each(function () { + var o = $(this); + if (!o.is(":checked")) return; + var tr = $(o.parents("tr").get(0)); + var q = o.attr("q"); + var remove_q; + + if (me.mode == "creator") { + var cmd = { q: q, status: "waiting", mode: "create" }; + var a = $($(tr.find("td").get(2)).find("a").get(0)); + if (a.hasClass("redlink")) cmd.from_redlink = true; + cmd.page = decodeURIComponent( + a.attr("href").replace(/^.+?\/wiki\//, ""), + ); + remove_q = me.commands_todo.length; + me.commands_todo.push(cmd); + } + + $.each(rows, function (k, row) { + let cmd = { q: q, status: "waiting" }; + let m = row.match(/^\s*D(\S+):(.+)$/i); + if (m != null) { + cmd.mode = "desc"; + cmd.language = m[1]; + cmd.value = m[2].replace(/^"(.+?)"$/, "$1"); + } else { + m = row.match(/^\s*-(P\d+)/i); + if (m == null) { + m = row.match(/^\s*(P\d+)\s*:\s*(Q\d+)\s*$/i); + if (m != null) { + cmd.mode = "add"; + cmd.prop = m[1]; + cmd.value = m[2]; + } else return; + } else { + // Delete property + cmd.mode = "delete"; + cmd.prop = m[1]; + m = row.match(/^\s*-(P\d+)\s*:\s*(Q\d+)/i); + if (m != null) cmd.value = m[2]; + } + } + remove_q = me.commands_todo.length; + me.commands_todo.push(cmd); + }); + if (typeof remove_q != "undefined") { + me.commands_todo[remove_q].remove_q = true; + me.commands_todo[remove_q].cb_id = o.attr("id"); + } + }); + }; + + this.finishCommand = function (id) { + var me = this; + me.commands_todo[id].status = "done"; + var cmd = me.commands_todo[id]; + + if (cmd.remove_q) { + // Last for this q, remove row + $( + $("#" + cmd.cb_id) + .parents("tr") + .get(0), + ).remove(); + } + + var tmp = []; + $.each(me.running, function (k, v) { + if (v != id) tmp.push(v); + }); + me.running = tmp; + + setTimeout(function () { + me.runNextCommand(); + }, me.delay); + }; + + this.addNewQ = function (q) { + if ($("#autolist_box_new_q").length == 0) { + $("#autolist_box").append( + "
", + ); + tt.updateInterface($("#autolist_box")); + } + var t = $("#autolist_box_new_q").val(); + if (t != "") t += "\n"; + t += "Q" + q; + $("#autolist_box_new_q").val(t); + }; + + this.runCommand = function (id) { + var me = this; + me.running.push(id); + me.commands_todo[id].status = "running"; + var cmd = me.commands_todo[id]; + + if (cmd.mode == "create") { + me.widar.createNewItem( + cmd.page, + output_wiki, + cmd.from_redlink, + function (d) { + // TODO error check: d.error=="OK" + if (typeof d == "undefined" || d.error != "OK") { + if (typeof d.error == "object") { + if (d.error.code == "no-external-page") { + console.log( + cmd.page + + " does not exist on " + + output_wiki + + "; maybe it has been deleted?", + ); + me.finishCommand(id); + return; + } + } + console.log(d); + } + + // Update all subsequent commands for this item to use the new Q + var new_q = d.q.replace(/\D/, ""); + var old_q = cmd.q; + me.addNewQ(new_q); + $.each(me.commands_todo, function (k, v) { + if (v.q == old_q) me.commands_todo[k].q = new_q; + }); + + // Next + me.finishCommand(id); + }, + ); + } else if (cmd.mode == "add") { + me.widar.setClaim("Q" + cmd.q, cmd.prop, cmd.value, function (d) { + // TODO error log + me.finishCommand(id); + }); + } else if (cmd.mode == "delete") { + $.getJSON( + "https://" + + autolist_wiki_server + + "/w/api.php?action=wbgetentities&ids=Q" + + cmd.q + + "&format=json&callback=?", + function (d) { + var done = false; + $.each( + (((d.entities || {})["Q" + cmd.q] || {}).claims || {})[ + cmd.prop.toUpperCase() + ] || {}, + function (k, v) { + if (typeof cmd.value != "undefined") { + // Specific value to delete + if ( + ((((v.mainsnak || {}).datavalue || {}).value || {})[ + "numeric-id" + ] || "") != cmd.value + ) + return; + } + done = true; + me.widar.removeClaim(v.id, function (d) { + // TODO error log + console.log(d); + me.finishCommand(id); + }); + return false; // Remove just the first one + }, + ); + if (!done) { + // TODO error log + me.finishCommand(id); + } + }, + ).fail(function () { + // TODO error log + me.finishCommand(id); + }); + } else { + console.log("Unknown mode: " + cmd.mode); + } + }; + + this.allDone = function () { + $("#al_do_stop").hide(); + $("#al_do_process").show(); + $("#al_start_qs").show(); + $("#al_status").html( + "" + _t("al_done") + "", + ); + }; + + this.runNextCommand = function () { + var me = this; + + //console.log ( me.concurrent , me.running.length ) ; + + if (me.emergency_stop) return; // Used clicked stop + + if (me.running.length >= me.concurrent) { + setTimeout(function () { + me.runNextCommand; + }, 100); // Was already called, so short delay + return; + } + + var q_running = {}; + $.each(me.running, function (k, v) { + q_running[me.commands_todo[v].q] = 1; + }); + + var run_next; + var has_commands_left = false; + $.each(me.commands_todo, function (k, v) { + if (v.status != "done") has_commands_left = true; + if (v.status != "waiting") return; // Already running + if (typeof q_running[v.q] != "undefined") return; // Don't run the same q multiple times + run_next = k; + return false; + }); + + if (!has_commands_left) { + me.allDone(); + return; + } + + if (typeof run_next == "undefined") { + setTimeout(function () { + me.runNextCommand; + }, 100); // Was already called, so short delay + return; + } + + me.updateRunCounter(); + me.runCommand(run_next); + }; + + this.updateRunCounter = function () { + var me = this; + var t = _t("al_running"); + var all = me.commands_todo.length; + var cnt = 0; + $.each(me.commands_todo, function (k, v) { + if (v.status != "done") cnt++; + }); + t = t.replace("$1", cnt); + t = t.replace("$2", all); + $("#al_status").html(t); + }; + + this.is_wikidata = function () { + return ( + typeof autolist_wiki_server == "undefined" || + autolist_wiki_server == "" || + autolist_wiki_server == "www.wikidata.org" + ); + }; + + this.is_commons = function () { + return autolist_wiki_server == "commons.wikimedia.org"; + }; + + this.initializeAutoListBox = function () { + var me = this; + var h = ""; + var p = getUrlVars(); + //if ( me.widar.isLoggedIn() ) { + h += "
"; + //h += "
" + _t('al_welcome').replace( '$1', me.widar.getUserName() ) + "
" ; + if (me.mode == "creator") { + h += "
"; + } + /* if ( me.widar.isBot() ) { me.max_concurrent = 5 ; me.concurrent = 5 ; @@ -353,171 +426,208 @@ function AutoList ( callback ) { h += "
(1-"+me.max_concurrent+")
" ; } */ - h += "
" ; - h += "
" ; - h += "
" ; - h += "
" ; - h += "
" ; - h += "
" ; - h += "
" ; - h += "
" ; - //console.log(autolist_wiki_server); - //if (this.is_wikidata()) h += "" ; - h += "" ; - h += "" ; - h += "" ; - h += "
" ; - h += "
" ; - //} else { - // h += "
" + me.widar.getLoginLink("") + "
" ; - //} - $('#autolist_box').html ( h ) ; - tt.updateInterface ( $('#autolist_box') ) ; - function updateConcurrency () { - var v = Math.floor ( $('#bot_concurrent').val() * 1 ) ; - if ( v < 1 || v > me.max_concurrent ) return ; - me.concurrent = v ; - } - - $('#bot_concurrent').keyup ( function(){updateConcurrency()} ) ; - $('#bot_concurrent').change ( function(){updateConcurrency()} ) ; - - if ( typeof p.al_commands != 'undefined' ) { - $('#al_commands').val ( p.al_commands ) ; - me.commandsHaveChanged() ; - } - - me.setInterfaceLanguage ( interface_language ) ; - - $('#al_start_qs').click ( function (e) { - e.preventDefault() ; - me.setupCommands() ; - let qs_commands = [] ; - let entity_letter = autolist_wiki_server=='www.wikidata.org'?'Q':'M'; - $.each ( me.commands_todo , function ( dummy , cmd ) { - let qs = '' ; - if ( cmd.mode == 'create' ) { - qs = 'CREATE' ; - qs += "||LAST|S" + output_wiki + "|\"" + cmd.page + "\"" ; - let m = output_wiki.match ( /^([a-z-]+)wiki$/ ) ; - let lang = m[1] ; - if ( lang == 'commons' ) lang = interface_language;//'en' ; - let the_label = $.trim(cmd.page.replace(/_/g,' ').replace(/\s*\(.+?\)\s*/,' ')).replace(/^\S+:/,''); - if ( m !== null ) qs += "||LAST|L" + lang + "|\"" + the_label + '"' ; - } else if ( cmd.mode == 'desc' ) { - qs = "LAST|D"+cmd.language+"|\""+cmd.value+"\""; - } else { - if ( cmd.mode == 'delete' ) qs = '-' ; - if ( /^create_item_/.test(cmd.q) ) qs += 'LAST' ; - else qs += entity_letter + cmd.q ; - qs += "|" + cmd.prop ; - if ( typeof cmd.value != 'undefined' ) { - if ( /^[PpQq]\d+$/.test(cmd.value) ) qs += "|" + cmd.value ; - else qs += "|" + cmd.value ; - } - } - - qs_commands.push ( qs ) ; - } ) ; - let s = qs_commands.join("||") ; - $('#qs_commands').val ( s ) ; - $('#qs_form').submit() ; - } ) ; - - $('#al_do_process').click ( function (e) { - e.preventDefault() ; - me.emergency_stop = false ; - $('#al_do_process').hide() ; - $('#al_do_stop').show() ; - me.setupCommands() ; - me.updateRunCounter() ; - - for ( var i = 0 ; i < me.concurrent ; i++ ) { - me.runNextCommand() ; - } - } ) ; - - $('#al_do_stop').click ( function (e) { - e.preventDefault() ; - me.emergency_stop = true ; - $('#al_do_process').show() ; - $('#al_do_stop').hide() ; - $('#al_status').html ( "" + _t('al_stopped') + "" ) ; - } ) ; - - $('#al_do_check_all').click ( function (e) { - e.preventDefault() ; - $('#main_table input.qcb').prop('checked',true) ; - } ) ; - - $('#al_do_check_none').click ( function (e) { - e.preventDefault() ; - $('#main_table input.qcb').prop('checked',false) ; - } ) ; - - $('#al_do_check_toggle').click ( function (e) { - e.preventDefault() ; - $('#main_table input.qcb').each ( function () { - var o = $(this) ; - o.prop ( 'checked' , !o.prop('checked') ) ; - } ) ; - } ) ; - - $('#al_commands').keyup ( function () { - me.commandsHaveChanged() ; - } ) ; - - } - - this.commandsHaveChanged = function () { - $('#main_form input[name="al_commands"]').remove() ; - $('#main_form').append ( "" ) ; - var v = $('#al_commands').val() ; - $('#main_form input[name="al_commands"]').val ( v ) ; - $('#permalink a').each ( function () { - var a = $(this) ; - var href = a.attr('href') ; - href = href.replace ( /&al_commands=[^&]+/ , '' ) ; - href += '&al_commands=' + encodeURIComponent ( v ) ; - href = href.replace ( /&{2,}/ , '&' ) ; - a.attr ( { href : href } ) ; - } ) ; - } - - this.setInterfaceLanguage = function ( l ) { - tt.setLanguage ( l ) ; - } - - this.addCheckLinks = function () { - var me = this ; - if ( me.mode != 'creator' ) return ; - $('#main_table tbody tr').each ( function () { - var tr = $(this) ; - var td = $(tr.find('td').get(2)) ; - var page = $(td.find('a').get(0)).attr('href').replace(/^.+?\/wiki\//,'').replace(/_/,' ') ; - td.append ( " []" ) ; - tt.updateInterface ( td ) ; - td.find('a[tt="check_wd"]').attr({href:'https://tools.wmflabs.org/wikidata-todo/duplicity.php?wiki='+output_wiki+'&norand=1&page='+page}) ; - } ) ; - } - - var me = this ; - if ( $('#autolist_box').length == 0 || $('#main_table input.qcb').length == 0 ) { // Don't bother - if ( typeof callback != 'undefined' ) callback() ; - return ; - } - - $( document ).ready(function(){ - me.mode = $('#autolist_box').attr('mode') ; - me.commands_todo = [] ; - me.addCheckLinks () ; - me.initializeAutoListBox() ; - }); -} + h += "
"; + h += "
"; + h += + "
"; + h += + "
"; + h += + "
"; + h += "
"; + h += "
"; + h += + "
"; + //console.log(autolist_wiki_server); + //if (this.is_wikidata()) h += "" ; + h += + ""; + h += + ""; + h += + ""; + h += "
"; + h += "
"; + //} else { + // h += "
" + me.widar.getLoginLink("") + "
" ; + //} + $("#autolist_box").html(h); + tt.updateInterface($("#autolist_box")); + function updateConcurrency() { + var v = Math.floor($("#bot_concurrent").val() * 1); + if (v < 1 || v > me.max_concurrent) return; + me.concurrent = v; + } + + $("#bot_concurrent").keyup(function () { + updateConcurrency(); + }); + $("#bot_concurrent").change(function () { + updateConcurrency(); + }); + if (typeof p.al_commands != "undefined") { + $("#al_commands").val(p.al_commands); + me.commandsHaveChanged(); + } + + me.setInterfaceLanguage(interface_language); + + $("#al_start_qs").click(function (e) { + e.preventDefault(); + me.setupCommands(); + let qs_commands = []; + let entity_letter = + autolist_wiki_server == "www.wikidata.org" ? "Q" : "M"; + $.each(me.commands_todo, function (dummy, cmd) { + let qs = ""; + if (cmd.mode == "create") { + qs = "CREATE"; + qs += "||LAST|S" + output_wiki + '|"' + cmd.page + '"'; + let m = output_wiki.match(/^([a-z-]+)wiki$/); + let lang = m[1]; + if (lang == "commons") lang = interface_language; //'en' ; + let the_label = $.trim( + cmd.page.replace(/_/g, " ").replace(/\s*\(.+?\)\s*/, " "), + ).replace(/^\S+:/, ""); + if (m !== null) qs += "||LAST|L" + lang + '|"' + the_label + '"'; + } else if (cmd.mode == "desc") { + qs = "LAST|D" + cmd.language + '|"' + cmd.value + '"'; + } else { + if (cmd.mode == "delete") qs = "-"; + if (/^create_item_/.test(cmd.q)) qs += "LAST"; + else qs += entity_letter + cmd.q; + qs += "|" + cmd.prop; + if (typeof cmd.value != "undefined") { + if (/^[PpQq]\d+$/.test(cmd.value)) qs += "|" + cmd.value; + else qs += "|" + cmd.value; + } + } + + qs_commands.push(qs); + }); + let s = qs_commands.join("||"); + $("#qs_commands").val(s); + $("#qs_form").submit(); + }); + + $("#al_do_process").click(function (e) { + e.preventDefault(); + me.emergency_stop = false; + $("#al_do_process").hide(); + $("#al_do_stop").show(); + me.setupCommands(); + me.updateRunCounter(); + + for (var i = 0; i < me.concurrent; i++) { + me.runNextCommand(); + } + }); + + $("#al_do_stop").click(function (e) { + e.preventDefault(); + me.emergency_stop = true; + $("#al_do_process").show(); + $("#al_do_stop").hide(); + $("#al_status").html( + "" + _t("al_stopped") + "", + ); + }); + + $("#al_do_check_all").click(function (e) { + e.preventDefault(); + $("#main_table input.qcb").prop("checked", true); + }); + + $("#al_do_check_none").click(function (e) { + e.preventDefault(); + $("#main_table input.qcb").prop("checked", false); + }); + + $("#al_do_check_toggle").click(function (e) { + e.preventDefault(); + $("#main_table input.qcb").each(function () { + var o = $(this); + o.prop("checked", !o.prop("checked")); + }); + }); + + $("#al_commands").keyup(function () { + me.commandsHaveChanged(); + }); + }; + + this.commandsHaveChanged = function () { + $('#main_form input[name="al_commands"]').remove(); + $("#main_form").append(""); + var v = $("#al_commands").val(); + $('#main_form input[name="al_commands"]').val(v); + $("#permalink a").each(function () { + var a = $(this); + var href = a.attr("href"); + href = href.replace(/&al_commands=[^&]+/, ""); + href += "&al_commands=" + encodeURIComponent(v); + href = href.replace(/&{2,}/, "&"); + a.attr({ href: href }); + }); + }; + + this.setInterfaceLanguage = function (l) { + tt.setLanguage(l); + }; + + this.addCheckLinks = function () { + var me = this; + if (me.mode != "creator") return; + $("#main_table tbody tr").each(function () { + var tr = $(this); + var td = $(tr.find("td").get(2)); + var page = $(td.find("a").get(0)) + .attr("href") + .replace(/^.+?\/wiki\//, "") + .replace(/_/, " "); + td.append( + " []", + ); + tt.updateInterface(td); + td.find('a[tt="check_wd"]').attr({ + href: + "https://tools.wmflabs.org/wikidata-todo/duplicity.php?wiki=" + + output_wiki + + "&norand=1&page=" + + page, + }); + }); + }; + + var me = this; + if ( + $("#autolist_box").length == 0 || + $("#main_table input.qcb").length == 0 + ) { + // Don't bother + if (typeof callback != "undefined") callback(); + return; + } + + $(document).ready(function () { + me.mode = $("#autolist_box").attr("mode"); + me.commands_todo = []; + me.addCheckLinks(); + me.initializeAutoListBox(); + }); +} diff --git a/html/main.js b/html/main.js index 785388b..4ed8529 100644 --- a/html/main.js +++ b/html/main.js @@ -1,630 +1,722 @@ -var tt = {} ; // Translations from tools-static.wmflabs.org/tooltranslate/tt.js -var params = {} ; -var ores_data = {} ; +var tt = {}; // Translations from tools-static.wmflabs.org/tooltranslate/tt.js +var params = {}; +var ores_data = {}; var default_params = { - 'show_redirects':'both', - 'show_soft_redirects':'both', - 'show_disambiguation_pages':'both', - 'edits[bots]':'both', - 'edits[anons]':'both', - 'edits[flagged]':'both', - 'page_image':'any', - 'ores_type':'any', - 'ores_prediction':'any', - 'combination':'subset', - 'wpiu':'any', - 'format':'html', - 'sortby':'none', - 'depth':'0', - 'min_redlink_count':'1', - 'wikidata_item':'no', - 'output_compatability':'catscan', - 'active_tab':'tab_categories', - 'common_wiki':'auto', - 'subpage_filter':'either', - 'namespace_conversion':'keep', - 'sortorder':'ascending' -} ; - -var thumbnail_size = '300px' ; -var max_namespace_id = 0 ; -var namespaces_selected = [] ; -var namespaces_loading = false ; -var last_namespaces = {} ; -var last_namespace_project = '' ; -var interface_language = '' ; - -var load_thumbnails_ahead = 2 ; // 1 to not load ahead -$.fn.is_on_screen = function(){ - var win = $(window); - var viewport = { - top : win.scrollTop(), - left : win.scrollLeft() - }; - viewport.right = viewport.left + win.width(); - viewport.bottom = viewport.top + win.height(); - - var bounds = this.offset(); - bounds.right = bounds.left + this.outerWidth(); - bounds.bottom = bounds.top + this.outerHeight()*load_thumbnails_ahead; - - return (!(viewport.right < bounds.left || viewport.left > bounds.right || viewport.bottom < bounds.top || viewport.top > bounds.bottom)); + show_redirects: "both", + show_soft_redirects: "both", + show_disambiguation_pages: "both", + "edits[bots]": "both", + "edits[anons]": "both", + "edits[flagged]": "both", + page_image: "any", + ores_type: "any", + ores_prediction: "any", + combination: "subset", + wpiu: "any", + format: "html", + sortby: "none", + depth: "0", + min_redlink_count: "1", + wikidata_item: "no", + output_compatability: "catscan", + active_tab: "tab_categories", + common_wiki: "auto", + subpage_filter: "either", + namespace_conversion: "keep", + sortorder: "ascending", }; -function getUrlVars () { - var vars = {} ; - var params = $('#querystring').text() ; - if ( params == '' ) params = window.location.href.slice(window.location.href.indexOf('?') + 1) ; - var hashes = params.split('&'); - if ( hashes.length >0 && hashes[0] == window.location.href ) hashes.shift() ; - $.each ( hashes , function ( i , j ) { - var hash = j.split('='); - hash[1] += '' ; - let value = decodeURIComponent(hash[1]) ; - if ( hash[0] != 'language' ) value.replace(/_/g,' ') ; - vars[decodeURIComponent(hash[0])] = value; - } ) ; - return vars; -} +var thumbnail_size = "300px"; +var max_namespace_id = 0; +var namespaces_selected = []; +var namespaces_loading = false; +var last_namespaces = {}; +var last_namespace_project = ""; +var interface_language = ""; + +var load_thumbnails_ahead = 2; // 1 to not load ahead +$.fn.is_on_screen = function () { + var win = $(window); + var viewport = { + top: win.scrollTop(), + left: win.scrollLeft(), + }; + viewport.right = viewport.left + win.width(); + viewport.bottom = viewport.top + win.height(); + + var bounds = this.offset(); + bounds.right = bounds.left + this.outerWidth(); + bounds.bottom = bounds.top + this.outerHeight() * load_thumbnails_ahead; + + return !( + viewport.right < bounds.left || + viewport.left > bounds.right || + viewport.bottom < bounds.top || + viewport.top > bounds.bottom + ); +}; -function _t ( k , alt_lang ) { - return tt.t(k,{lang:alt_lang}) ; +function getUrlVars() { + var vars = {}; + var params = $("#querystring").text(); + if (params == "") + params = window.location.href.slice(window.location.href.indexOf("?") + 1); + var hashes = params.split("&"); + if (hashes.length > 0 && hashes[0] == window.location.href) hashes.shift(); + $.each(hashes, function (i, j) { + var hash = j.split("="); + hash[1] += ""; + let value = decodeURIComponent(hash[1]); + if (hash[0] != "language") value.replace(/_/g, " "); + vars[decodeURIComponent(hash[0])] = value; + }); + return vars; } -function setPermalink () { - var psid = 0 ; - $('span[name="psid"]').each ( function () { psid = $(this).text() ; } ) ; - - var q = $("#querystring").text(); - if ( q == '' ) return; - - const params = new URLSearchParams(q); - // Removing auto-run - params.delete("doit"); - params.forEach(function (value, key) { - // Removing empty parameters - if (value === "") params.delete(key); - // Removing default values - if (value === default_params[key]) params.delete(key); - }); - - var url = '/?' + params ; - var h = _t("query_url") ; - if ( typeof h == 'undefined' ) return ; - h = h.replace ( /\$1/ , url+"&doit=" ) ; - h = h.replace ( /\$2/ , url ) ; - - if ( psid != 0 ) { - var psid_note = (_t("psid_note")).split('$1').join(psid) ;// ( /\$1/ , psid ) ; - h += ' ' ; - h += psid_note ; - h += " " ; - h += _t("psid_image_link") ; - h += "." ; - } - - $('#permalink').html ( h ) ; +function _t(k, alt_lang) { + return tt.t(k, { lang: alt_lang }); } -function applyParameters () { - - namespaces_selected = [] ; - - $.each ( params , function ( name , value ) { - - var m = name.match ( /^ns\[(\d+)\]$/ ) ; - if ( m != null ) { - namespaces_selected.push ( m[1]*1 ) ; - return ; - } - - $('input:radio[name="'+name+'"][value="'+value.replace(/"/g,'"')+'"]').prop('checked', true); - - $('input[type="hidden"][name="'+name+'"]').val ( value ) ; - $('input[type="text"][name="'+name+'"]').val ( value ) ; - $('input[type="number"][name="'+name+'"]').val ( parseInt(value) ) ; - $('textarea[name="'+name+'"]').val ( value ) ; - - if ( value == '1' || value == 'on' ) $('input[type="checkbox"][name="'+name+'"]').prop('checked', true); - - } ) ; - - if ( params['doit'] != '' && typeof params['referrer_url'] != 'undefined' && params['referrer_url'] != '' && params['psid'] != '' ) { - var psid = 0 ; - $('span[name="psid"]').each ( function () { psid = $(this).text() ; } ) ; - if ( psid != 0 ) { - let url = params['referrer_url'].replace('{PSID}',psid) ; - let name = params['referrer_name'] || url; - $('#referrer').attr({href:url}).text(name); - $("#referrer_box").show(); - } - } - - function wait2load_ns () { - if ( namespaces_loading ) { - setTimeout ( wait2load_ns , 100 ) ; - return ; - } - last_namespace_project = '' ; - - loadNamespaces() ; - } - wait2load_ns() ; - - // Permalink - setPermalink () ; - - if ( typeof params.active_tab != 'undefined' ) { - var tab = '#' + params.active_tab.replace(/ /g,'_') ; //$('input[name="active_tab"]').val() ; - $('#main_form ul.nav-tabs a[href="'+tab+'"]').click() ; - } - - $('body').show() ; +function setPermalink() { + var psid = 0; + $('span[name="psid"]').each(function () { + psid = $(this).text(); + }); + + var q = $("#querystring").text(); + if (q == "") return; + + const params = new URLSearchParams(q); + // Removing auto-run + params.delete("doit"); + params.forEach(function (value, key) { + // Removing empty parameters + if (value === "") params.delete(key); + // Removing default values + if (value === default_params[key]) params.delete(key); + }); + + var url = "/?" + params; + var h = _t("query_url"); + if (typeof h == "undefined") return; + h = h.replace(/\$1/, url + "&doit="); + h = h.replace(/\$2/, url); + + if (psid != 0) { + var psid_note = _t("psid_note").split("$1").join(psid); // ( /\$1/ , psid ) ; + h += " "; + h += psid_note; + h += + " "; + h += _t("psid_image_link"); + h += "."; + } + + $("#permalink").html(h); } -function setInterfaceLanguage ( l ) { - if ( interface_language == l ) return false ; - interface_language = l ; - - function specialUpdates () { - // Misc special updates - $('a[tt="manual"]').attr ( { href:'https://meta.wikimedia.org/wiki/PetScan/'+l } ) ; - $('#query_length').text ( _t('query_length').replace('$1',$('#query_length').attr('sec')) ) ; - $('#num_results').text ( _t('num_results').replace('$1',$('#num_results').attr('num')) ) ; - - // Permalink - setPermalink () ; - - $('#permalink a').each ( function () { - var a = $(this) ; - var h = a.attr('href') ; - var h2 = h.replace ( /\binterface_language=[^&]+/ , 'interface_language='+l ) ; - if ( h == h2 ) h2 = h + "&interface_language=" + l ; - a.attr ( { href : h2 } ) ; - } ) ; - } - - - if ( tt.language != l ) { - tt.setLanguage ( l , function () { - specialUpdates() ; - } ) ; - } else { - specialUpdates() ; - } - - - - return false ; +function applyParameters() { + namespaces_selected = []; + + $.each(params, function (name, value) { + var m = name.match(/^ns\[(\d+)\]$/); + if (m != null) { + namespaces_selected.push(m[1] * 1); + return; + } + + $( + 'input:radio[name="' + + name + + '"][value="' + + value.replace(/"/g, """) + + '"]', + ).prop("checked", true); + + $('input[type="hidden"][name="' + name + '"]').val(value); + $('input[type="text"][name="' + name + '"]').val(value); + $('input[type="number"][name="' + name + '"]').val(parseInt(value)); + $('textarea[name="' + name + '"]').val(value); + + if (value == "1" || value == "on") + $('input[type="checkbox"][name="' + name + '"]').prop("checked", true); + }); + + if ( + params["doit"] != "" && + typeof params["referrer_url"] != "undefined" && + params["referrer_url"] != "" && + params["psid"] != "" + ) { + var psid = 0; + $('span[name="psid"]').each(function () { + psid = $(this).text(); + }); + if (psid != 0) { + let url = params["referrer_url"].replace("{PSID}", psid); + let name = params["referrer_name"] || url; + $("#referrer").attr({ href: url }).text(name); + $("#referrer_box").show(); + } + } + + function wait2load_ns() { + if (namespaces_loading) { + setTimeout(wait2load_ns, 100); + return; + } + last_namespace_project = ""; + + loadNamespaces(); + } + wait2load_ns(); + + // Permalink + setPermalink(); + + if (typeof params.active_tab != "undefined") { + var tab = "#" + params.active_tab.replace(/ /g, "_"); //$('input[name="active_tab"]').val() ; + $('#main_form ul.nav-tabs a[href="' + tab + '"]').click(); + } + + $("body").show(); } -function loadInterface ( init_lang , callback ) { - - loadNamespaces ( function () { - - setInterfaceLanguage ( init_lang ) ; +function setInterfaceLanguage(l) { + if (interface_language == l) return false; + interface_language = l; + + function specialUpdates() { + // Misc special updates + $('a[tt="manual"]').attr({ + href: "https://meta.wikimedia.org/wiki/PetScan/" + l, + }); + $("#query_length").text( + _t("query_length").replace("$1", $("#query_length").attr("sec")), + ); + $("#num_results").text( + _t("num_results").replace("$1", $("#num_results").attr("num")), + ); + + // Permalink + setPermalink(); + + $("#permalink a").each(function () { + var a = $(this); + var h = a.attr("href"); + var h2 = h.replace( + /\binterface_language=[^&]+/, + "interface_language=" + l, + ); + if (h == h2) h2 = h + "&interface_language=" + l; + a.attr({ href: h2 }); + }); + } + + if (tt.language != l) { + tt.setLanguage(l, function () { + specialUpdates(); + }); + } else { + specialUpdates(); + } + + return false; +} - $('input[name="language"]').keyup ( function () { loadNamespaces() } ) ; - $('input[name="project"]').keyup ( function () { loadNamespaces() } ) ; +function loadInterface(init_lang, callback) { + loadNamespaces(function () { + setInterfaceLanguage(init_lang); + $('input[name="language"]').keyup(function () { + loadNamespaces(); + }); + $('input[name="project"]').keyup(function () { + loadNamespaces(); + }); - applyParameters() ; - if ( typeof callback != 'undefined' ) callback() ; - - } ) ; + applyParameters(); + if (typeof callback != "undefined") callback(); + }); } -function loadNamespaces ( callback ) { - if ( namespaces_loading ) return false ; - - var l = $('input[name="language"]').val() ; - if ( l.length < 2 ) return false ; - var p = $('input[name="project"]').val() ; - if ( l == 'wikidata' ) { l = 'www' ; p = 'wikidata' ; } - var lp = l+'.'+p ; - if ( lp == last_namespace_project ) return false ; - - // ORES - var wiki = l + 'wiki' ; - if ( p == 'wikidata' ) wiki = 'wikidatawiki' ; - else if ( p != 'wikipedia' && l != 'species' && l != 'commons' ) wiki = l + p ; - if ( typeof ores_data[wiki] == 'undefined' || typeof ores_data[wiki].models == 'undefined' ) { - $('#ores_options').hide() ; - } else { - var current = $('#ores_model_select').val() ; - var h = "" ; - $.each ( ores_data[wiki].models , function ( model , content ) { - h += "" ; - } ) ; - $('#ores_model_select').html ( h ) ; - $('#ores_model_select').val(current) ; - $('#ores_options').show() ; - tt.updateInterface ( $('#ores_options') ) ; - } - - - function namespaceDataLoaded ( d ) { - namespaces_loading = false ; - if ( typeof d == 'undefined' ) return ; - if ( typeof d.query == 'undefined' ) return ; - if ( typeof d.query.namespaces == 'undefined' ) return ; - last_namespaces = {} ; - max_namespace_id = 0 ; - $.each ( d.query.namespaces , function ( id , v ) { - id *= 1 ; - if ( id < 0 ) return ; - var title = v['*'] ; - if ( title == '' ) title = "" ; //_t('namespace_0',l) ; - last_namespaces[id] = [ title , (v.canonical||title) ] ; - if ( id > max_namespace_id ) max_namespace_id = id ; - } ) ; - - last_namespace_project = lp ; - - var nsl = [] ; - function renderNS ( ns ) { - var h = "
" ; - if ( typeof last_namespaces[ns] != 'undefined' ) { - h += "= 0 ) { - nsl.push ( ns ) ; - h += " checked" ; - } - h += "> " ; - h += last_namespaces[ns][0] ; - h += "" ; - } else h += "—" ; - h += "
" ; - return h ; - } - - $(".current_wiki").text ( lp ) ; - h = "" ; - h += "
" ; - for ( var ns = 0 ; ns <= max_namespace_id ; ns += 2 ) { - if ( typeof last_namespaces[ns] == 'undefined' && typeof last_namespaces[ns+1] == 'undefined' ) continue ; - h += "
" ; - h += renderNS ( ns ) ; - h += renderNS ( ns+1 ) ; - h += "
" ; - } - h += "
" ; - namespaces_selected = nsl ; - - $('#namespaces').html ( h ) ; - $('#namespaces input').change ( function () { - var o = $(this) ; - var ns = o.attr('ns') * 1 ; - namespaces_selected = jQuery.grep(namespaces_selected, function(value) { return value != ns; }); // Remove is present - if ( o.is(':checked') ) namespaces_selected.push ( ns ) ; - } ) ; - - } - - var server = lp+'.org' ; - if ( typeof global_namespace_cache[server] == 'undefined' ) { - namespaces_loading = true ; - $.getJSON ( 'https://'+server+'/w/api.php?action=query&meta=siteinfo&siprop=namespaces&format=json&callback=?' , function ( d ) { - global_namespace_cache[server] = d ; - namespaceDataLoaded ( d ) ; - } ) . always ( function () { - namespaces_loading = false ; - if ( typeof callback != 'undefined' ) callback() ; - } ) ; - } else { - namespaceDataLoaded ( global_namespace_cache[server] ) ; - namespaces_loading = false ; - if ( typeof callback != 'undefined' ) callback() ; - } - return false ; +function loadNamespaces(callback) { + if (namespaces_loading) return false; + + var l = $('input[name="language"]').val(); + if (l.length < 2) return false; + var p = $('input[name="project"]').val(); + if (l == "wikidata") { + l = "www"; + p = "wikidata"; + } + var lp = l + "." + p; + if (lp == last_namespace_project) return false; + + // ORES + var wiki = l + "wiki"; + if (p == "wikidata") wiki = "wikidatawiki"; + else if (p != "wikipedia" && l != "species" && l != "commons") wiki = l + p; + if ( + typeof ores_data[wiki] == "undefined" || + typeof ores_data[wiki].models == "undefined" + ) { + $("#ores_options").hide(); + } else { + var current = $("#ores_model_select").val(); + var h = ""; + $.each(ores_data[wiki].models, function (model, content) { + h += ""; + }); + $("#ores_model_select").html(h); + $("#ores_model_select").val(current); + $("#ores_options").show(); + tt.updateInterface($("#ores_options")); + } + + function namespaceDataLoaded(d) { + namespaces_loading = false; + if (typeof d == "undefined") return; + if (typeof d.query == "undefined") return; + if (typeof d.query.namespaces == "undefined") return; + last_namespaces = {}; + max_namespace_id = 0; + $.each(d.query.namespaces, function (id, v) { + id *= 1; + if (id < 0) return; + var title = v["*"]; + if (title == "") title = ""; //_t('namespace_0',l) ; + last_namespaces[id] = [title, v.canonical || title]; + if (id > max_namespace_id) max_namespace_id = id; + }); + + last_namespace_project = lp; + + var nsl = []; + function renderNS(ns) { + var h = "
"; + if (typeof last_namespaces[ns] != "undefined") { + h += "= 0) { + nsl.push(ns); + h += " checked"; + } + h += "> "; + h += last_namespaces[ns][0]; + h += ""; + } else h += "—"; + h += "
"; + return h; + } + + $(".current_wiki").text(lp); + h = ""; + h += "
"; + for (var ns = 0; ns <= max_namespace_id; ns += 2) { + if ( + typeof last_namespaces[ns] == "undefined" && + typeof last_namespaces[ns + 1] == "undefined" + ) + continue; + h += "
"; + h += renderNS(ns); + h += renderNS(ns + 1); + h += "
"; + } + h += "
"; + namespaces_selected = nsl; + + $("#namespaces").html(h); + $("#namespaces input").change(function () { + var o = $(this); + var ns = o.attr("ns") * 1; + namespaces_selected = jQuery.grep(namespaces_selected, function (value) { + return value != ns; + }); // Remove is present + if (o.is(":checked")) namespaces_selected.push(ns); + }); + } + + var server = lp + ".org"; + if (typeof global_namespace_cache[server] == "undefined") { + namespaces_loading = true; + $.getJSON( + "https://" + + server + + "/w/api.php?action=query&meta=siteinfo&siprop=namespaces&format=json&callback=?", + function (d) { + global_namespace_cache[server] = d; + namespaceDataLoaded(d); + }, + ).always(function () { + namespaces_loading = false; + if (typeof callback != "undefined") callback(); + }); + } else { + namespaceDataLoaded(global_namespace_cache[server]); + namespaces_loading = false; + if (typeof callback != "undefined") callback(); + } + return false; } -var autolist ; - +var autolist; // Converts a WDQ input box to SPARQL via wdq2sparql, if possible -function wdq2sparql ( wdq , sparql_selector ) { - $(sparql_selector).prop('disabled', true); - - $.get ( '/wdq2sparql/?' + encodeURIComponent(wdq) , function ( d ) { - $(sparql_selector).prop('disabled', false); - if ( d.status != 'OK' ) { - alert ( _t('wdq2sparql_fail') ) ; - return ; - } - var sparql = $.trim ( d.sparql ) ; - sparql = sparql.replace ( /^prefix.+$/mig , '' ) ; - sparql = sparql.replace ( /\s+/g , ' ' ) ; - sparql = sparql.replace ( /\s*\.\s*\}\s*$/g , ' }' ) ; - sparql = $.trim ( sparql ) ; - $(sparql_selector).val ( sparql ) ; - } , 'json' ) ; - - -/* - $.get ( 'https://tools.wmflabs.org/wdq2sparql/w2s.php' , { - wdq:wdq - } , function ( d ) { - $(sparql_selector).prop('disabled', false); - if ( d.match ( /^" ; - $('#main_table').after ( h ) ; - - $('#main_table tbody tr').each ( function () { - var tr = $(this) ; - var td = $(tr.find('td.link_container').get(0)) ; - var a = $(td.find('a.pagelink').get(0)) ; - var url = a.attr('href').replace(/\/wiki\/File(:|%3[aA])/,'/wiki/Special:Redirect/file/') + '?width=' + thumbnail_size ; - var h = '' ; - h += '
' ; - h += '
'+'??'+'
' ; - h += '
' ; - h += '

' ; - h += $('

').append(a.clone()).html(); - h += '

' ; - h += '
' ; - h += '
' ; - var card = $(h) ; - card.find('a.thumblink').attr({href:a.attr('href'),target:'_blank'}) ; - $('#thumbnails').append ( card ) ; - } ) ; - - // Load thumbnails on demand - $(window).scroll(function(){ - $('#thumbnails img.pre_thumb').each ( function () { - loadThumbnail ( $(this) ) ; - } ) ; - } ) ; + var h = "
"; + $("#main_table").after(h); + + $("#main_table tbody tr").each(function () { + var tr = $(this); + var td = $(tr.find("td.link_container").get(0)); + var a = $(td.find("a.pagelink").get(0)); + var url = + a + .attr("href") + .replace(/\/wiki\/File(:|%3[aA])/, "/wiki/Special:Redirect/file/") + + "?width=" + + thumbnail_size; + var h = ""; + h += '
'; + h += + '
' + +
'; + h += '
'; + h += '

'; + h += $("

").append(a.clone()).html(); + h += "

"; + h += "
"; + h += "
"; + var card = $(h); + card.find("a.thumblink").attr({ href: a.attr("href"), target: "_blank" }); + $("#thumbnails").append(card); + }); + + // Load thumbnails on demand + $(window).scroll(function () { + $("#thumbnails img.pre_thumb").each(function () { + loadThumbnail($(this)); + }); + }); } -function validateSourceCombination () { - var o = $('input[name="source_combination"]') ; - var text = $.trim ( o.val() ) ; - var reg = /^( and | or | not |categories|sparql|pagepile|manual|wikidata|search|\(|\)| )*$/gi ; - var op = $(o.parents("div.input-group").get(0)) ; - if ( reg.test ( text ) ) { - op.removeClass ( 'has-danger' ) ; - $('#doit').prop('disabled', false) - } else { - op.addClass ( 'has-danger' ) ; - $('#doit').prop('disabled', true) - } - return false ; +function validateSourceCombination() { + var o = $('input[name="source_combination"]'); + var text = $.trim(o.val()); + var reg = + /^( and | or | not |categories|sparql|pagepile|manual|wikidata|search|\(|\)| )*$/gi; + var op = $(o.parents("div.input-group").get(0)); + if (reg.test(text)) { + op.removeClass("has-danger"); + $("#doit").prop("disabled", false); + } else { + op.addClass("has-danger"); + $("#doit").prop("disabled", true); + } + return false; } -var example_list = [] ; - -function showExamples ( filter ) { - var h = '' ; - $('#example_list').html ( h ) ; - $.each ( example_list , function ( dummy , example ) { - if ( filter != '' ) { - var desc = example.desc.toLowerCase() ; - var words = filter.toLowerCase().split ( /\s+/ ) ; - var found = 0 ; - var max_words = 0 ; - $.each ( words , function ( k , v ) { - if ( v == '' ) return ; - max_words++ ; - if ( null !== desc.match ( v ) ) found++ ; - } ) ; - if ( found != max_words ) return ; - } - var row = $("
"); - row.html("
" + example.psid + "
"); - row.append($("
").text(example.desc)); - $('#example_list').append(row); - } ) ; - - if ( filter != '' ) return ; - - $('#example_dialog').modal ( 'show' ) ; +var example_list = []; + +function showExamples(filter) { + var h = ""; + $("#example_list").html(h); + $.each(example_list, function (dummy, example) { + if (filter != "") { + var desc = example.desc.toLowerCase(); + var words = filter.toLowerCase().split(/\s+/); + var found = 0; + var max_words = 0; + $.each(words, function (k, v) { + if (v == "") return; + max_words++; + if (null !== desc.match(v)) found++; + }); + if (found != max_words) return; + } + var row = $("
"); + row.html( + "
" + + example.psid + + "
", + ); + row.append($("
").text(example.desc)); + $("#example_list").append(row); + }); + + if (filter != "") return; + + $("#example_dialog").modal("show"); } -function addExamples () { -// $('#example_dialog').on('shown.bs.modal',function(){$('#example_search').focus(); return false}) ; - var o = $('a[tt="examples"]') ; - o.click ( function () { - $('#example_search').val('') ; - if ( example_list.length == 0 ) { - $.getJSON('https://meta.wikimedia.org/w/api.php?action=parse&prop=wikitext&page=PetScan/Examples&format=json&callback=?',function(d){ - var rows = d.parse.wikitext['*'].split("\n") ; - $.each ( rows , function ( dummy , row ) { - var m = row.match ( /^;\s*(\d+)\s*:(.*)$/ ) ; - if ( m === null ) return ; - example_list.push ( { psid:m[1] , desc:m[2] } ) ; - } ) ; - showExamples(''); - } ) ; - } else { - showExamples(''); - } - return false ; - } ) ; - - $('#example_search').keyup ( function () { - var filter = $('#example_search').val() ; - showExamples ( filter ) ; - } ) ; +function addExamples() { + // $('#example_dialog').on('shown.bs.modal',function(){$('#example_search').focus(); return false}) ; + var o = $('a[tt="examples"]'); + o.click(function () { + $("#example_search").val(""); + if (example_list.length == 0) { + $.getJSON( + "https://meta.wikimedia.org/w/api.php?action=parse&prop=wikitext&page=PetScan/Examples&format=json&callback=?", + function (d) { + var rows = d.parse.wikitext["*"].split("\n"); + $.each(rows, function (dummy, row) { + var m = row.match(/^;\s*(\d+)\s*:(.*)$/); + if (m === null) return; + example_list.push({ psid: m[1], desc: m[2] }); + }); + showExamples(""); + }, + ); + } else { + showExamples(""); + } + return false; + }); + + $("#example_search").keyup(function () { + var filter = $("#example_search").val(); + showExamples(filter); + }); } -function initializeInterface () { - var p = getUrlVars() ; - - - // Ensure NS0 is selected by default - var cnt = 0 ; - $.each ( p , function ( k , v ) { cnt++ } ) ; - if ( cnt == 0 ) p['ns[0]'] = 1 ; - - // Legacy parameters - if ( typeof p.category != 'undefined' && (p.categories||'') == '' ) p.categories = p.category ; - if ( typeof p.cats != 'undefined' && (p.categories||'') == '' ) p.categories = p.cats ; - if ( typeof p.wdqs != 'undefined' ) p.sparql = p.wdqs ; - if ( typeof p.statementlist != 'undefined' ) p.al_commands = p.statementlist ; - if ( typeof p.comb_subset != 'undefined' ) p.combination = 'subset' ; - if ( typeof p.comb_union != 'undefined' ) p.combination = 'union' ; - if ( typeof p.get_q != 'undefined' ) p.wikidata_item = 'any' ; - if ( typeof p.wikidata != 'undefined' ) p.wikidata_item = 'any' ; - if ( typeof p.wikidata_no_item != 'undefined' ) p.wikidata_item = 'without' ; - if ( typeof p.giu != 'undefined' ) p.file_usage_data = 'on' ; - - params = $.extend ( {} , default_params , p ) ; - params.sortby = params.sortby.replace ( / /g , '_' ) ; - $('#ores_model_select').val(params.ores_type) ; - - $.each(["yes","any","no"],function(k,v){ - let base = 'cb_labels_'+v+"_" ; - if ( (params[base+"l"]||'')=='' && (params[base+"a"]||'')=='' && (params[base+"d"]||'')=='' ) { - params[base+"l"]="1"; - } - }); - - - var l = 'en' ; - if ( typeof params.interface_language != 'undefined' ) l = params.interface_language ; - - loadInterface ( l , function () { - if ( typeof window.AutoList != 'undefined' ) autolist = new AutoList () ; - } ) ; - $('input[name="language"]').focus() ; - - $('#main_form ul.nav-tabs a').click ( function (e) { - e.preventDefault() ; - var o = $(this) ; - $('input[name="active_tab"]').val ( o.attr('href').replace(/^\#/,'') ) ; - } ) ; - - $('#wdq2sparql').click ( function ( e ) { - e.preventDefault() ; - var wdq = prompt ( _t('wdq2sparql_prompt') , '' ) ; - if ( wdq == null || $.trim(wdq) == '' ) return false ; - wdq2sparql ( wdq , 'textarea[name="sparql"]' ) ; - return false ; - } ) ; - - $('input[name="source_combination"]').keyup ( validateSourceCombination ) ; - - addExamples() ; - - $('#file_results label').change ( function () { - var o = $('#file_results input[name="results_mode"]:checked') ; - var mode = o.val() ; - - if ( mode == 'titles' ) { - $('#thumbnails').hide() ; - $('#main_table').show() ; - return ; - } - - // mode == 'thumbnails' - - if ( $('#thumbnails').length == 0 ) { - generateThumbnailView() ; - } - - $('#main_table').hide() ; - $('#thumbnails').show() ; - $(window).scroll() ; - } ) ; - - function highlightMissingWiki ( n1 , n2 ) { - var o = $('[name="'+n1+'"]') ; - var wo = $('input[name="'+n2+'"]') ; - var text = o.val() ; - var wiki = wo.val() ; -// var wop = $(wo.parents("div.input-group").get(0)) ; - if ( $.trim(text) != '' && !wiki.match(/(wiki|source|quote)\s*$/) ) { //$.trim(wiki) == '' ) { - wo.addClass ( 'is-invalid' ) ; - $('#doit').prop('disabled', true) - } else { - wo.removeClass ( 'is-invalid' ) ; - $('#doit').prop('disabled', false) - } - - if ( n1 == 'common_wiki_other' ) { - $('input:radio[name=common_wiki]')[5].checked = true; - } - } - $('[name="manual_list"]').keyup ( function () {highlightMissingWiki('manual_list','manual_list_wiki')} ) ; - $('[name="search_query"]').keyup ( function () {highlightMissingWiki('search_query','search_wiki')} ) ; - $('[name="manual_list_wiki"]').keyup ( function () {highlightMissingWiki('manual_list','manual_list_wiki')} ) ; - $('[name="search_wiki"]').keyup ( function () {highlightMissingWiki('search_query','search_wiki')} ) ; - $('[name="common_wiki_other"]').keyup ( function () {highlightMissingWiki('common_wiki_other','common_wiki_other')} ) ; - highlightMissingWiki('manual_list','manual_list_wiki') ; - highlightMissingWiki('search_query','search_wiki'); - - $('#tab-list').click ( function () { - if ( $('#main_form div.tab-pane').length > 0 ) { - $('#tab-list').text ( _t('toggle_tabs') ) ; - $('#main_form ul.nav-tabs').hide() ; - $('#main_form div.tab-pane').addClass('former-tab-pane').removeClass('tab-pane') ; - } else { - $('#tab-list').text ( _t('toggle_list') ) ; - $('#main_form ul.nav-tabs').show() ; - $('#main_form div.former-tab-pane').addClass('tab-pane').removeClass('former-tab-pane') ; - } - } ) ; - - // Deactivate REDIRECTS when showing only pages without item. This will be a mess otherwise, few users would think to change that setting. It can always be changed back manually. - $('input[name="wikidata_item"][value="without"]').click ( function ( e ) { - $('input[name="show_redirects"][value="no"]').prop('checked', true); - } ) ; - - $('#quick_commons').click ( function () { $("input[name='language']").val("commons"); $("input[name='project']").val("wikimedia"); loadNamespaces() ; return false } ) ; - $('#quick_wikispecies').click ( function () { $("input[name='language']").val("species"); $("input[name='project']").val("wikimedia"); loadNamespaces() ; return false } ) ; - $('#quick_wikidata').click ( function () { $("input[name='language']").val("wikidata"); $("input[name='project']").val("wikimedia"); loadNamespaces() ; return false } ) ; - +function initializeInterface() { + var p = getUrlVars(); + + // Ensure NS0 is selected by default + var cnt = 0; + $.each(p, function (k, v) { + cnt++; + }); + if (cnt == 0) p["ns[0]"] = 1; + + // Legacy parameters + if (typeof p.category != "undefined" && (p.categories || "") == "") + p.categories = p.category; + if (typeof p.cats != "undefined" && (p.categories || "") == "") + p.categories = p.cats; + if (typeof p.wdqs != "undefined") p.sparql = p.wdqs; + if (typeof p.statementlist != "undefined") p.al_commands = p.statementlist; + if (typeof p.comb_subset != "undefined") p.combination = "subset"; + if (typeof p.comb_union != "undefined") p.combination = "union"; + if (typeof p.get_q != "undefined") p.wikidata_item = "any"; + if (typeof p.wikidata != "undefined") p.wikidata_item = "any"; + if (typeof p.wikidata_no_item != "undefined") p.wikidata_item = "without"; + if (typeof p.giu != "undefined") p.file_usage_data = "on"; + + params = $.extend({}, default_params, p); + params.sortby = params.sortby.replace(/ /g, "_"); + $("#ores_model_select").val(params.ores_type); + + $.each(["yes", "any", "no"], function (k, v) { + let base = "cb_labels_" + v + "_"; + if ( + (params[base + "l"] || "") == "" && + (params[base + "a"] || "") == "" && + (params[base + "d"] || "") == "" + ) { + params[base + "l"] = "1"; + } + }); + + var l = "en"; + if (typeof params.interface_language != "undefined") + l = params.interface_language; + + loadInterface(l, function () { + if (typeof window.AutoList != "undefined") autolist = new AutoList(); + }); + $('input[name="language"]').focus(); + + $("#main_form ul.nav-tabs a").click(function (e) { + e.preventDefault(); + var o = $(this); + $('input[name="active_tab"]').val(o.attr("href").replace(/^\#/, "")); + }); + + $("#wdq2sparql").click(function (e) { + e.preventDefault(); + var wdq = prompt(_t("wdq2sparql_prompt"), ""); + if (wdq == null || $.trim(wdq) == "") return false; + wdq2sparql(wdq, 'textarea[name="sparql"]'); + return false; + }); + + $('input[name="source_combination"]').keyup(validateSourceCombination); + + addExamples(); + + $("#file_results label").change(function () { + var o = $('#file_results input[name="results_mode"]:checked'); + var mode = o.val(); + + if (mode == "titles") { + $("#thumbnails").hide(); + $("#main_table").show(); + return; + } + + // mode == 'thumbnails' + + if ($("#thumbnails").length == 0) { + generateThumbnailView(); + } + + $("#main_table").hide(); + $("#thumbnails").show(); + $(window).scroll(); + }); + + function highlightMissingWiki(n1, n2) { + var o = $('[name="' + n1 + '"]'); + var wo = $('input[name="' + n2 + '"]'); + var text = o.val(); + var wiki = wo.val(); + // var wop = $(wo.parents("div.input-group").get(0)) ; + if ($.trim(text) != "" && !wiki.match(/(wiki|source|quote)\s*$/)) { + //$.trim(wiki) == '' ) { + wo.addClass("is-invalid"); + $("#doit").prop("disabled", true); + } else { + wo.removeClass("is-invalid"); + $("#doit").prop("disabled", false); + } + + if (n1 == "common_wiki_other") { + $("input:radio[name=common_wiki]")[5].checked = true; + } + } + $('[name="manual_list"]').keyup(function () { + highlightMissingWiki("manual_list", "manual_list_wiki"); + }); + $('[name="search_query"]').keyup(function () { + highlightMissingWiki("search_query", "search_wiki"); + }); + $('[name="manual_list_wiki"]').keyup(function () { + highlightMissingWiki("manual_list", "manual_list_wiki"); + }); + $('[name="search_wiki"]').keyup(function () { + highlightMissingWiki("search_query", "search_wiki"); + }); + $('[name="common_wiki_other"]').keyup(function () { + highlightMissingWiki("common_wiki_other", "common_wiki_other"); + }); + highlightMissingWiki("manual_list", "manual_list_wiki"); + highlightMissingWiki("search_query", "search_wiki"); + + $("#tab-list").click(function () { + if ($("#main_form div.tab-pane").length > 0) { + $("#tab-list").text(_t("toggle_tabs")); + $("#main_form ul.nav-tabs").hide(); + $("#main_form div.tab-pane") + .addClass("former-tab-pane") + .removeClass("tab-pane"); + } else { + $("#tab-list").text(_t("toggle_list")); + $("#main_form ul.nav-tabs").show(); + $("#main_form div.former-tab-pane") + .addClass("tab-pane") + .removeClass("former-tab-pane"); + } + }); + + // Deactivate REDIRECTS when showing only pages without item. This will be a mess otherwise, few users would think to change that setting. It can always be changed back manually. + $('input[name="wikidata_item"][value="without"]').click(function (e) { + $('input[name="show_redirects"][value="no"]').prop("checked", true); + }); + + $("#quick_commons").click(function () { + $("input[name='language']").val("commons"); + $("input[name='project']").val("wikimedia"); + loadNamespaces(); + return false; + }); + $("#quick_wikispecies").click(function () { + $("input[name='language']").val("species"); + $("input[name='project']").val("wikimedia"); + loadNamespaces(); + return false; + }); + $("#quick_wikidata").click(function () { + $("input[name='language']").val("wikidata"); + $("input[name='project']").val("wikimedia"); + loadNamespaces(); + return false; + }); } -$(document).ready ( function () { - var running = 2 ; - function fin () { - running-- ; - if ( running > 0 ) return ; - tt.addILdropdown ( '#interface_languages_wrapper' ) ; - initializeInterface() ; - } - tt = new ToolTranslation ( { - tool:'petscan', - language:'en', - fallback:'en', - highlight_missing:true, - callback:function () { - fin() ; - } , - onUpdateInterface : function () { - setInterfaceLanguage ( tt.language ) ; - } - } ) ; - - fetch("https://ores.wikimedia.org/v3/scores/") - .then((response) => response.json()) - .then((data) => { - ores_data = data ; - fin(); - }) - .catch(fin); -} ) ; \ No newline at end of file +$(document).ready(function () { + var running = 2; + function fin() { + running--; + if (running > 0) return; + tt.addILdropdown("#interface_languages_wrapper"); + initializeInterface(); + } + tt = new ToolTranslation({ + tool: "petscan", + language: "en", + fallback: "en", + highlight_missing: true, + callback: function () { + fin(); + }, + onUpdateInterface: function () { + setInterfaceLanguage(tt.language); + }, + }); + + fetch("https://ores.wikimedia.org/v3/scores/") + .then((response) => response.json()) + .then((data) => { + ores_data = data; + fin(); + }) + .catch(fin); +}); diff --git a/src/app_state.rs b/src/app_state.rs index 2b33992..22408ea 100644 --- a/src/app_state.rs +++ b/src/app_state.rs @@ -153,8 +153,12 @@ impl AppState { /// Returns the server and database name for the wiki, as a tuple pub fn db_host_and_schema_for_wiki(&self, wiki: &str) -> Result<(String, String), String> { // TESTING - // ssh magnus@tools-login.wmflabs.org -L 3307:dewiki.web.db.svc.eqiad.wmflabs:3306 -N - // ssh magnus@tools-login.wmflabs.org -L 3309:wikidatawiki.web.db.svc.eqiad.wmflabs:3306 -N + /* + ssh magnus@tools-login.wmflabs.org -L 3307:dewiki.web.db.svc.eqiad.wmflabs:3306 -N & + ssh magnus@tools-login.wmflabs.org -L 3309:wikidatawiki.web.db.svc.eqiad.wmflabs:3306 -N & + ssh magnus@tools-login.wmflabs.org -L 3305:commonswiki.web.db.svc.eqiad.wmflabs:3306 -N & + ssh magnus@tools-login.wmflabs.org -L 3310:enwiki.web.db.svc.eqiad.wmflabs:3306 -N & + */ let wiki = self.fix_wiki_name(wiki); let host = match self.config["host"].as_str() { Some("127.0.0.1") => "127.0.0.1".to_string(), @@ -168,7 +172,7 @@ impl AppState { /// Returns the server and database name for the tool db, as a tuple pub fn db_host_and_schema_for_tool_db(&self) -> (String, String) { // TESTING - // ssh magnus@tools-login.wmflabs.org -L 3308:tools-db:3306 -N + // ssh magnus@tools-login.wmflabs.org -L 3308:tools-db:3306 -N & let host = self.config["host"] .as_str() .expect("No host key in config file") diff --git a/src/datasource_database.rs b/src/datasource_database.rs index aab5083..12877b3 100644 --- a/src/datasource_database.rs +++ b/src/datasource_database.rs @@ -1355,16 +1355,17 @@ mod tests { assert!(result.len() > result_size2); } - #[tokio::test] - async fn test_category_case_sensitive() { - let params = vec![ - ("categories", "français de France"), - ("language", "fr"), - ("project", "wiktionary"), - ]; - let result = simulate_category_query(params).await.unwrap(); - assert!(result.len().unwrap() > 0); - } + // Deactivated: connection to frwiktionary_p required + // #[tokio::test] + // async fn test_category_case_sensitive() { + // let params = vec![ + // ("categories", "français de France"), + // ("language", "fr"), + // ("project", "wiktionary"), + // ]; + // let result = simulate_category_query(params).await.unwrap(); + // assert!(result.len().unwrap() > 0); + // } #[tokio::test] async fn test_category_case_insensitive() { diff --git a/src/platform.rs b/src/platform.rs index 9df7f97..8e89332 100644 --- a/src/platform.rs +++ b/src/platform.rs @@ -2154,7 +2154,11 @@ mod tests { check_results_for_psid( 10225056, "wikidatawiki", - vec![Title::new("Q13520818", 0), Title::new("Q10995651", 0)], + vec![ + Title::new("Q13520818", 0), + Title::new("Q10995651", 0), + Title::new("Q20084080", 0), + ], ) .await; } @@ -2207,7 +2211,7 @@ mod tests { assert!(entry.get_page_timestamp().is_some()); assert_eq!( entry.get_page_image(), - Some("KingsCollegeChapelWest.jpg".to_string()) + Some("Kings_College_(233225593).jpeg".to_string()) ); assert_eq!(entry.disambiguation, TriState::No); assert!(entry.incoming_links.is_some()); @@ -2327,10 +2331,11 @@ mod tests { .await; } - #[tokio::test] - async fn test_en_categories_sparql_common_wiki_other() { - check_results_for_psid(15960820, "frwiki", vec![Title::new("Magnus Manske", 0)]).await; - } + // Deactivated: connection to frwiki_p required + // #[tokio::test] + // async fn test_en_categories_sparql_common_wiki_other() { + // check_results_for_psid(15960820, "frwiki", vec![Title::new("Magnus Manske", 0)]).await; + // } fn entries_from_result(result: PageList) -> Vec { result @@ -2342,13 +2347,14 @@ mod tests { .collect::>() } - #[tokio::test] - async fn test_trim_extended_whitespace() { - let platform = run_psid(15015735).await; // The categories contain a left-to-right mark - let result = platform.result.unwrap(); - let entries = entries_from_result(result); - assert!(entries.len() > 20); - } + // Deactivated: connection to enwikiquote_p required + // #[tokio::test] + // async fn test_trim_extended_whitespace() { + // let platform = run_psid(15015735).await; // The categories contain a left-to-right mark + // let result = platform.result.unwrap(); + // let entries = entries_from_result(result); + // assert!(entries.len() > 20); + // } #[tokio::test] async fn test_template_talk_pages() { diff --git a/src/wdfist.rs b/src/wdfist.rs index b6bbca6..08136ba 100644 --- a/src/wdfist.rs +++ b/src/wdfist.rs @@ -174,48 +174,8 @@ impl WDfist { let add_item_file: Mutex> = Mutex::new(vec![]); let wiki2title_q = self.get_language_links().await?; for (wiki, title_q) in wiki2title_q { - // Prepare batches - let page2q: HashMap = title_q - .par_iter() - .map(|(title, q)| (title.to_string(), q.to_string())) - .collect(); - let titles: Vec = page2q - .par_iter() - .map(|(title, _q)| title.to_string()) - .collect(); - let mut batches: Vec = vec![]; - titles.chunks(PAGE_BATCH_SIZE).for_each(|chunk| { - let mut sql = Platform::prep_quote(chunk); - sql.0 = format!("SELECT DISTINCT gil_page_title AS page,gil_to AS image FROM page,globalimagelinks WHERE gil_wiki='{}' AND gil_page_title IN ({})",wiki,&sql.0) ; - sql.0 += " AND gil_page_namespace_id=0 AND page_namespace=6 and page_title=gil_to AND page_is_redirect=0" ; - sql.0 += " AND NOT EXISTS (SELECT * FROM categorylinks where page_id=cl_from and cl_to='Crop_for_Wikidata')" ; // To-be-cropped - batches.push(sql); - }); - - // Run batches - let rows = PageList::new_from_wiki("commonswiki") - .run_batch_queries(&self.state, batches) - .await - .map_err(|e| format!("{:?}", e))?; - - // Collect pages and items, per wiki - let page_file: Vec<(String, String)> = rows - .par_iter() - .map(|row| my::from_row::<(String, String)>(row.to_owned())) - .collect(); - let mut page_file = self - .filter_page_images(&wiki, page_file) - .await - .map_err(|e| format!("{:?}", e))? - .par_iter() - .filter_map(|(page, file)| { - page2q.get(page).map(|q| (q.to_string(), file.to_string())) - }) - .collect(); - add_item_file - .lock() - .map_err(|e| format!("{:?}", e))? - .append(&mut page_file); + self.follow_language_links_wiki2title_q(title_q, wiki, &add_item_file) + .await?; } // Add files @@ -228,6 +188,50 @@ impl WDfist { Ok(()) } + async fn follow_language_links_wiki2title_q( + &mut self, + title_q: Vec<(String, String)>, + wiki: String, + add_item_file: &Mutex>, + ) -> Result<(), String> { + let page2q: HashMap = title_q + .par_iter() + .map(|(title, q)| (title.to_string(), q.to_string())) + .collect(); + let titles: Vec = page2q + .par_iter() + .map(|(title, _q)| title.to_string()) + .collect(); + let mut batches: Vec = vec![]; + titles.chunks(PAGE_BATCH_SIZE).for_each(|chunk| { + let mut sql = Platform::prep_quote(chunk); + sql.0 = format!("SELECT DISTINCT gil_page_title AS page,gil_to AS image FROM page,globalimagelinks WHERE gil_wiki='{}' AND gil_page_title IN ({})",wiki,&sql.0) ; + sql.0 += " AND gil_page_namespace_id=0 AND page_namespace=6 and page_title=gil_to AND page_is_redirect=0" ; + sql.0 += " AND NOT EXISTS (SELECT * FROM categorylinks where page_id=cl_from and cl_to='Crop_for_Wikidata')" ; // To-be-cropped + batches.push(sql); + }); + let rows = PageList::new_from_wiki("commonswiki") + .run_batch_queries(&self.state, batches) + .await + .map_err(|e| format!("{:?}", e))?; + let page_file: Vec<(String, String)> = rows + .par_iter() + .map(|row| my::from_row::<(String, String)>(row.to_owned())) + .collect(); + let mut page_file = self + .filter_page_images(&wiki, page_file) + .await + .map_err(|e| format!("{:?}", e))? + .par_iter() + .filter_map(|(page, file)| page2q.get(page).map(|q| (q.to_string(), file.to_string()))) + .collect(); + add_item_file + .lock() + .map_err(|e| format!("{:?}", e))? + .append(&mut page_file); + Ok(()) + } + async fn follow_coords(&mut self) -> Result<(), String> { // Prepare batches let mut batches: Vec = vec![]; @@ -248,36 +252,8 @@ impl WDfist { .collect(); // Get nearby files - let api = Api::new("https://commons.wikimedia.org/w/api.php") - .await - .map_err(|e| format!("{:?}", e))?; - //let add_item_file: Mutex> = Mutex::new(vec![]); - - let params: Vec<_> = page_coords - .iter() - .map(|(_q, lat, lon)| { - api.params_into(&[ - ("action", "query"), - ("list", "geosearch"), - ("gscoord", format!("{}|{}", lat, lon).as_str()), - ( - "gsradius", - format!("{}", NEARBY_FILES_RADIUS_IN_METERS).as_str(), - ), - ("gslimit", "50"), - ("gsnamespace", "6"), - ]) - }) - .collect(); - - /* - let futures : Vec<_> = params - .iter() - .map(|params|api.get_query_api_json(¶ms)) - .collect(); - - let results = join_all(futures).await; - */ + let api = self.get_commons_api().await?; + let params = self.follow_coords_get_params(&page_coords, &api); let mut results: Vec<_> = vec![]; for param in params { @@ -287,6 +263,19 @@ impl WDfist { } } + let add_item_file = self.follow_coords_get_item_files(results, page_coords); + add_item_file + .iter() + .for_each(|(q, file)| self.add_file_to_item(q, file)); + + Ok(()) + } + + fn follow_coords_get_item_files( + &mut self, + results: Vec, + page_coords: Vec<(String, f64, f64)>, + ) -> Vec<(String, String)> { let add_item_file: Vec<(String, String)> = results .iter() .zip(page_coords) @@ -311,13 +300,7 @@ impl WDfist { }) .flatten() .collect(); - - // Add files add_item_file - .iter() - .for_each(|(q, file)| self.add_file_to_item(q, file)); - - Ok(()) } // Remove leading "File:" @@ -342,6 +325,26 @@ impl WDfist { } async fn follow_search_commons(&mut self) -> Result<(), String> { + let batches = self.follow_search_commons_prepare_batches(); + let pagelist = PageList::new_from_wiki("wikidatawiki"); + let rows = pagelist.run_batch_queries(&self.state, batches).await?; + let item2label = self.follow_search_commons_get_item2label(rows); + + // Get search results + let api = self.get_commons_api().await?; + let params = self.follow_search_commons_get_params(&item2label, &api); + let results = self.follow_search_commons_get_results(params, api).await; + let add_item_file = self.get_add_item_files(results, item2label); + + // Add files + add_item_file + .iter() + .for_each(|(q, file)| self.add_file_to_item(q, file)); + + Ok(()) + } + + fn follow_search_commons_prepare_batches(&mut self) -> Vec<(String, Vec)> { // Prepare batches let mut batches: Vec = vec![]; self.items.chunks(PAGE_BATCH_SIZE).for_each(|chunk| { @@ -349,22 +352,14 @@ impl WDfist { sql.0 = format!("SELECT concat('Q',wbit_item_id) AS term_full_entity_id, wbx_text as term_text FROM wbt_item_terms INNER JOIN wbt_term_in_lang ON wbit_term_in_lang_id = wbtl_id INNER JOIN wbt_type ON wbtl_type_id = wby_id AND wby_name='label' INNER JOIN wbt_text_in_lang ON wbtl_text_in_lang_id = wbxl_id INNER JOIN wbt_text ON wbxl_text_id = wbx_id AND wbxl_language='en' WHERE wbit_item_id IN ({})",&sql.0) ; batches.push(sql); }); + batches + } - // Run batches - let pagelist = PageList::new_from_wiki("wikidatawiki"); - let rows = pagelist.run_batch_queries(&self.state, batches).await?; - - // Process results - let item2label: Vec<(String, String)> = rows - .par_iter() - .map(|row| my::from_row::<(String, String)>(row.to_owned())) - .collect(); - - // Get search results - let api = Api::new("https://commons.wikimedia.org/w/api.php") - .await - .map_err(|e| format!("{:?}", e))?; - + fn follow_search_commons_get_params( + &mut self, + item2label: &[(String, String)], + api: &Api, + ) -> Vec> { let params: Vec<_> = item2label .iter() .map(|(_q, label)| { @@ -376,24 +371,14 @@ impl WDfist { ]) }) .collect(); + params + } - /* - let futures : Vec<_> = params - .iter() - .map(|params|api.get_query_api_json(¶ms)) - .collect(); - - let results = join_all(futures).await; - */ - - let mut results: Vec<_> = vec![]; - for param in params { - match api.get_query_api_json(¶m).await { - Ok(x) => results.push(x), - _ => results.push(json!({})), // Ignore - } - } - + fn get_add_item_files( + &mut self, + results: Vec, + item2label: Vec<(String, String)>, + ) -> Vec<(String, String)> { let add_item_file: Vec<(String, String)> = results .iter() .zip(item2label) @@ -423,13 +408,7 @@ impl WDfist { }) .flatten() .collect(); - - // Add files add_item_file - .iter() - .for_each(|(q, file)| self.add_file_to_item(q, file)); - - Ok(()) } fn follow_commons_cats(&mut self) -> Result<(), String> { @@ -481,11 +460,7 @@ impl WDfist { } async fn seed_ignore_files_from_ignore_database(&mut self) -> Result<(), String> { - let state = self.state.clone(); - let tool_db_user_pass = state.get_tool_db_user_pass().lock().await; - let mut conn = state - .get_tool_db_connection(tool_db_user_pass.clone()) - .await?; + let mut conn = self.get_db_conn().await?; let sql = format!("SELECT CONVERT(`file` USING utf8) FROM s51218__wdfist_p.ignore_files GROUP BY file HAVING count(*)>={}",MIN_IGNORE_DB_FILE_COUNT); @@ -543,6 +518,27 @@ impl WDfist { return Ok(()); } + let batches = self.filter_files_from_ignore_database_prepare_batches(); + let mut conn = self.get_db_conn().await?; + + // Run batches sequentially + for sql in batches { + self.filter_files_from_ignore_database_run_batch(&mut conn, sql) + .await?; + } + Ok(()) + } + + async fn get_db_conn(&mut self) -> Result { + let state = self.state.clone(); + let tool_db_user_pass = state.get_tool_db_user_pass().lock().await; + let conn = state + .get_tool_db_connection(tool_db_user_pass.clone()) + .await?; + Ok(conn) + } + + fn filter_files_from_ignore_database_prepare_batches(&mut self) -> Vec<(String, Vec)> { // Prepare batches let mut batches: Vec = vec![]; let items: Vec = self @@ -558,31 +554,27 @@ impl WDfist { ); batches.push(sql); }); + batches + } - // Prepare - let state = self.state.clone(); - let tool_db_user_pass = state.get_tool_db_user_pass().lock().await; - let mut conn = state - .get_tool_db_connection(tool_db_user_pass.clone()) - .await?; - - // Run batches sequentially - for sql in batches { - let rows = conn - .exec_iter(sql.0.as_str(), mysql_async::Params::Positional(sql.1)) - .await - .map_err(|e| format!("{:?}", e))? - .map_and_drop(from_row::<(String, String)>) - .await - .map_err(|e| format!("{:?}", e))?; - - for (item, filename) in rows { - let filename = self.normalize_filename(&filename.to_string()); - if let Some(ref mut files) = self.item2files.get_mut(&item) { - files.remove(&filename); - if files.is_empty() { - self.item2files.remove(&item); - } + async fn filter_files_from_ignore_database_run_batch( + &mut self, + conn: &mut my::Conn, + sql: (String, Vec), + ) -> Result<(), String> { + let rows = conn + .exec_iter(sql.0.as_str(), mysql_async::Params::Positional(sql.1)) + .await + .map_err(|e| format!("{:?}", e))? + .map_and_drop(from_row::<(String, String)>) + .await + .map_err(|e| format!("{:?}", e))?; + for (item, filename) in rows { + let filename = self.normalize_filename(&filename.to_string()); + if let Some(ref mut files) = self.item2files.get_mut(&item) { + files.remove(&filename); + if files.is_empty() { + self.item2files.remove(&item); } } } @@ -594,46 +586,47 @@ impl WDfist { return Ok(()); } - // Collect all filenames, and how often they are used in this result set - let mut file2count: HashMap = HashMap::new(); - self.item2files.iter().for_each(|(_item, files)| { - files - .iter() - .for_each(|fc| *file2count.entry(fc.0.to_string()).or_insert(0) += 1); - }); + let file2count = self.get_file_counts(); if file2count.is_empty() { return Ok(()); } let mut files_to_remove: Vec = vec![]; + self.wdf_only_files_not_on_wd(&file2count, &mut files_to_remove) + .await?; + self.remove_max_files_returned(&mut files_to_remove, file2count); + self.remove_files(files_to_remove); - if self.bool_param("wdf_only_files_not_on_wd") { - // Get distinct filenames to check - let filenames: Vec = file2count - .par_iter() - .map(|(file, _count)| file.to_owned()) - .collect(); - - // Create batches - let mut batches: Vec = vec![]; - filenames.chunks(PAGE_BATCH_SIZE).for_each(|chunk| { - let mut sql = Platform::prep_quote(chunk); - sql.0 = format!( - "SELECT DISTINCT il_to FROM imagelinks WHERE il_from_namespace=0 AND il_to IN ({})", - &sql.0 - ); - batches.push(sql); + // Remove empty item results + self.item2files.retain(|_item, files| !files.is_empty()); + Ok(()) + } + + fn remove_files(&mut self, files_to_remove: Vec) { + // Remove the files + self.item2files.iter_mut().for_each(|(_item, files)| { + files_to_remove.iter().for_each(|filename| { + files.remove(filename); }); + }); + } - // Run batches, and get a list of files to remove - let pagelist = PageList::new_from_wiki("wikidatawiki"); - let rows = pagelist.run_batch_queries(&self.state, batches).await?; - files_to_remove = rows - .par_iter() - .map(|row| my::from_row::(row.to_owned())) - .collect(); - } + fn get_file_counts(&mut self) -> HashMap { + // Collect all filenames, and how often they are used in this result set + let mut file2count: HashMap = HashMap::new(); + self.item2files.iter().for_each(|(_item, files)| { + files + .iter() + .for_each(|fc| *file2count.entry(fc.0.to_string()).or_insert(0) += 1); + }); + file2count + } + fn remove_max_files_returned( + &mut self, + files_to_remove: &mut Vec, + file2count: HashMap, + ) { // Remove max files returned if self.bool_param("wdf_max_five_results") { files_to_remove.extend( @@ -645,17 +638,40 @@ impl WDfist { files_to_remove.par_sort(); files_to_remove.dedup(); } + } - // Remove the files - self.item2files.iter_mut().for_each(|(_item, files)| { - files_to_remove.iter().for_each(|filename| { - files.remove(filename); - }); - }); + async fn wdf_only_files_not_on_wd( + &mut self, + file2count: &HashMap, + files_to_remove: &mut Vec, + ) -> Result<(), String> { + if !self.bool_param("wdf_only_files_not_on_wd") { + return Ok(()); + } + // Get distinct filenames to check + let filenames: Vec = file2count + .par_iter() + .map(|(file, _count)| file.to_owned()) + .collect(); - // Remove empty item results - self.item2files.retain(|_item, files| !files.is_empty()); + // Create batches + let mut batches: Vec = vec![]; + filenames.chunks(PAGE_BATCH_SIZE).for_each(|chunk| { + let mut sql = Platform::prep_quote(chunk); + sql.0 = format!( + "SELECT DISTINCT il_to FROM imagelinks WHERE il_from_namespace=0 AND il_to IN ({})", + &sql.0 + ); + batches.push(sql); + }); + // Run batches, and get a list of files to remove + let pagelist = PageList::new_from_wiki("wikidatawiki"); + let rows = pagelist.run_batch_queries(&self.state, batches).await?; + *files_to_remove = rows + .par_iter() + .map(|row| my::from_row::(row.to_owned())) + .collect(); Ok(()) } @@ -728,6 +744,60 @@ impl WDfist { } } } + + async fn follow_search_commons_get_results( + &self, + params: Vec>, + api: Api, + ) -> Vec { + let mut results: Vec<_> = vec![]; + for param in params { + match api.get_query_api_json(¶m).await { + Ok(x) => results.push(x), + _ => results.push(json!({})), // Ignore + } + } + results + } + + async fn get_commons_api(&self) -> Result { + let api = Api::new("https://commons.wikimedia.org/w/api.php") + .await + .map_err(|e| format!("{:?}", e))?; + Ok(api) + } + + fn follow_search_commons_get_item2label(&self, rows: Vec) -> Vec<(String, String)> { + let item2label: Vec<(String, String)> = rows + .par_iter() + .map(|row| my::from_row::<(String, String)>(row.to_owned())) + .collect(); + item2label + } + + fn follow_coords_get_params( + &self, + page_coords: &[(String, f64, f64)], + api: &Api, + ) -> Vec> { + let params: Vec<_> = page_coords + .iter() + .map(|(_q, lat, lon)| { + api.params_into(&[ + ("action", "query"), + ("list", "geosearch"), + ("gscoord", format!("{}|{}", lat, lon).as_str()), + ( + "gsradius", + format!("{}", NEARBY_FILES_RADIUS_IN_METERS).as_str(), + ), + ("gslimit", "50"), + ("gsnamespace", "6"), + ]) + }) + .collect(); + params + } } #[cfg(test)] @@ -850,38 +920,39 @@ mod tests { assert!(!wdfist.is_valid_filename(&"foobar.svg".to_string())); } - #[tokio::test] - async fn test_follow_language_links() { - let params: Vec<(&str, &str)> = vec![]; - let mut wdfist = get_wdfist(params, vec!["Q1481"]).await; - - // All files - wdfist.wdf_allow_svg = true; - wdfist.follow_language_links().await.unwrap(); - assert!(wdfist.item2files.contains_key(&"Q1481".to_string())); - assert!(wdfist.item2files.get(&"Q1481".to_string()).unwrap().len() > 90); - - // No SVG - wdfist.item2files.clear(); - wdfist.wdf_allow_svg = false; - wdfist.follow_language_links().await.unwrap(); - assert!(wdfist.item2files.contains_key(&"Q1481".to_string())); - assert!(wdfist.item2files.get(&"Q1481".to_string()).unwrap().len() < 50); - assert!(wdfist - .item2files - .get(&"Q1481".to_string()) - .unwrap() - .contains_key(&"Felsberg_bl_von_turm_zwei_wv_ds_09_2006.JPG".to_string())); - - // Page images - let params: Vec<(&str, &str)> = vec![("wdf_only_page_images", "1")]; - let mut wdfist = get_wdfist(params, vec!["Q1481"]).await; - wdfist.follow_language_links().await.unwrap(); - assert!(wdfist.item2files.contains_key(&"Q1481".to_string())); - let x = wdfist.item2files.get(&"Q1481".to_string()).unwrap(); - assert!(x.len() < 50); - assert!(x.contains_key(&"Felsberg_(Hessen).jpg".to_string())); - } + // Deactivated: connection to cewiki_p required + // #[tokio::test] + // async fn test_follow_language_links() { + // let params: Vec<(&str, &str)> = vec![]; + // let mut wdfist = get_wdfist(params, vec!["Q1481"]).await; + + // // All files + // wdfist.wdf_allow_svg = true; + // wdfist.follow_language_links().await.unwrap(); + // assert!(wdfist.item2files.contains_key(&"Q1481".to_string())); + // assert!(wdfist.item2files.get(&"Q1481".to_string()).unwrap().len() > 90); + + // // No SVG + // wdfist.item2files.clear(); + // wdfist.wdf_allow_svg = false; + // wdfist.follow_language_links().await.unwrap(); + // assert!(wdfist.item2files.contains_key(&"Q1481".to_string())); + // assert!(wdfist.item2files.get(&"Q1481".to_string()).unwrap().len() < 50); + // assert!(wdfist + // .item2files + // .get(&"Q1481".to_string()) + // .unwrap() + // .contains_key(&"Felsberg_bl_von_turm_zwei_wv_ds_09_2006.JPG".to_string())); + + // // Page images + // let params: Vec<(&str, &str)> = vec![("wdf_only_page_images", "1")]; + // let mut wdfist = get_wdfist(params, vec!["Q1481"]).await; + // wdfist.follow_language_links().await.unwrap(); + // assert!(wdfist.item2files.contains_key(&"Q1481".to_string())); + // let x = wdfist.item2files.get(&"Q1481".to_string()).unwrap(); + // assert!(x.len() < 50); + // assert!(x.contains_key(&"Felsberg_(Hessen).jpg".to_string())); + // } #[tokio::test] async fn test_follow_coords() { @@ -915,6 +986,14 @@ mod tests { .item2files .get(&"Q66711783".to_string()) .unwrap() - .contains_key(&"Gerst_lilly_nee_kohn_with_son_walter_samuel.png".to_string())); + .contains_key(&"Walter_Archer_and_family.jpg".to_string())); + } + + #[tokio::test] + async fn test_seed_ignore_files() { + let params: Vec<(&str, &str)> = vec![]; + let mut wdfist = get_wdfist(params, vec![]).await; + wdfist.seed_ignore_files().await.unwrap(); + assert!(wdfist.files2ignore.len() > 100); } }