diff --git a/build.xml b/build.xml index 62d374a470..37a91d5e8d 100644 --- a/build.xml +++ b/build.xml @@ -12,7 +12,7 @@ # ========================================== # This Software is Copyrighted # (C) by Michael Peter Christen; mc@yacy.net -# first published on http://yacy.net +# first published on https://yacy.net # Frankfurt, Germany, 2005-2009 # # This program is free software; you can redistribute it and/or modify diff --git a/defaults/oaiListFriendsSource.xml b/defaults/oaiListFriendsSource.xml index 838686b072..3b41b2a3e8 100644 --- a/defaults/oaiListFriendsSource.xml +++ b/defaults/oaiListFriendsSource.xml @@ -2,6 +2,6 @@ Sources for OAI ListFriends entries. The Key is the URL to the ListFriends Source, the Value is the Path to the local file where the ListFriends file is stored - DATA/DICTIONARIES/harvesting/export_roar_ROAR_ListFriends.xml - DATA/DICTIONARIES/harvesting/ListFriends.xml + DATA/DICTIONARIES/harvesting/export_roar_ROAR_ListFriends.xml + DATA/DICTIONARIES/harvesting/ListFriends.xml \ No newline at end of file diff --git a/defaults/yacy.init b/defaults/yacy.init index 68e3e72cca..eac5bef474 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -597,7 +597,7 @@ proxyIndexingLocalMedia=true # proxy usage only for .yacy-Domains for autoconfig proxyYacyOnly=false -# enable proxy via url (/proxy.html?url=http://yacy.net) +# enable proxy via url (/proxy.html?url=https://yacy.net) proxyURL=false proxyURL.access=127.0.0.1,0:0:0:0:0:0:0:1 # which urls to rewrite to /proxy.html?url=x (values: all, domainlist) @@ -1288,8 +1288,8 @@ content.phpbb3.dumpfile = # search engine teaser: an about box in search results # this is only shown, if the about.body is filled -about.headline= -about.body= +about.headline=Please Help Us +about.body= donation.iframesource=https://yacy.net/include/donate.html donation.iframetarget=env/donate.html @@ -1405,7 +1405,7 @@ postprocessing.partialUpdate = true # To use this user agent option, you must define completely different names and strings # and remove the given example here, which will be ignored by default. crawler.userAgent.name = yacybot -crawler.userAgent.string = yacybot ($$SYSTEM$$) http://yacy.net/bot.html +crawler.userAgent.string = yacybot ($$SYSTEM$$) https://yacy.net/bot.html crawler.userAgent.minimumdelta = 500 crawler.userAgent.clienttimeout = 10000 diff --git a/examples/SimpleSearchClient/src/YaCySearchClient.java b/examples/SimpleSearchClient/src/YaCySearchClient.java index 0c2a6f5352..7c8f11b95f 100644 --- a/examples/SimpleSearchClient/src/YaCySearchClient.java +++ b/examples/SimpleSearchClient/src/YaCySearchClient.java @@ -2,7 +2,7 @@ * YaCySearchClient * an interface for Adaptive Replacement Caches * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 20.09.2010 at http://yacy.net + * First released 20.09.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/htroot/api/yacydoc.xml b/htroot/api/yacydoc.xml index 10ed039882..887abbc2a3 100644 --- a/htroot/api/yacydoc.xml +++ b/htroot/api/yacydoc.xml @@ -6,7 +6,7 @@ this is a xml file with embedded dublin core properties you can validate it with http://www.stg.brown.edu/service/xmlvalid/ --> - + #[dc_title]# #[dc_creator]# diff --git a/htroot/env/grafics/donate.html b/htroot/env/grafics/donate.html new file mode 100644 index 0000000000..d01b4bf5f7 --- /dev/null +++ b/htroot/env/grafics/donate.html @@ -0,0 +1,41 @@ + + + + + +Please support our work on YaCy! + + + + + + + + + + + + + + + + + + + +
Github Sponsors
+ + + + + + +
+
+ + diff --git a/htroot/js/Crawler.js b/htroot/js/Crawler.js index c2f77404e4..3e840ac235 100644 --- a/htroot/js/Crawler.js +++ b/htroot/js/Crawler.js @@ -1,6 +1,6 @@ /* * Copyright (C) 2005 - 2014 Alexander Schier, Michael Peter Christen, -* and other YaCy developers (see http://yacy.net/en/Join.html) +* and other YaCy developers (see https://yacy.net/en/Join.html) * * This file is part of YaCy. * @@ -15,7 +15,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License -* along with YaCy. If not, see . +* along with YaCy. If not, see . */ DELETE_STRING="delete"; diff --git a/htroot/portalsearch/yacy-portalsearch.js b/htroot/portalsearch/yacy-portalsearch.js index a1f80cffba..ad499ed91f 100644 --- a/htroot/portalsearch/yacy-portalsearch.js +++ b/htroot/portalsearch/yacy-portalsearch.js @@ -50,7 +50,7 @@ $(document).ready(function() { theme : 'start', title : 'YaCy Search Widget', logo : yconf.url + '/yacy/ui/img/yacy-logo.png', - link : 'http://yacy.net', + link : 'https://yacy.net', width : 640, height : 640, position : [150,50], diff --git a/htroot/yacysearch.atom b/htroot/yacysearch.atom index 4513f8ce26..21f2f29f69 100644 --- a/htroot/yacysearch.atom +++ b/htroot/yacysearch.atom @@ -4,7 +4,7 @@ xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:yacy="http://www.yacy.net/" > - + #[promoteSearchPageGreeting]#: #[rss_query]# Search for #[rss_query]# diff --git a/htroot/yacysearch.rss b/htroot/yacysearch.rss index 9513be15a7..c85a744e7c 100644 --- a/htroot/yacysearch.rss +++ b/htroot/yacysearch.rss @@ -9,7 +9,7 @@ xmlns:geo="http://www.w3.org/2003/01/geo/wgs84_pos#" xmlns:ev="http://purl.org/rss/1.0/modules/event/" > - + #[promoteSearchPageGreeting]#: #[rss_query]# Search for #[rss_query]# diff --git a/htroot/yacysearch_location.rss b/htroot/yacysearch_location.rss index e58547c123..8311f87726 100644 --- a/htroot/yacysearch_location.rss +++ b/htroot/yacysearch_location.rss @@ -4,7 +4,7 @@ xmlns:yacy="http://www.yacy.net/" xmlns:geo="http://www.w3.org/2003/01/geo/wgs84_pos#" > - + #[promoteSearchPageGreeting]# Search for #[rss_query]# diff --git a/source/net/yacy/ai/OllamaClient.java b/source/net/yacy/ai/OllamaClient.java index 742645fee3..9eaba920b4 100644 --- a/source/net/yacy/ai/OllamaClient.java +++ b/source/net/yacy/ai/OllamaClient.java @@ -1,7 +1,7 @@ /** * OllamaClient * Copyright 2024 by Michael Peter Christen - * First released 17.05.2024 at http://yacy.net + * First released 17.05.2024 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -35,33 +35,33 @@ public class OllamaClient { public static String OLLAMA_API_HOST = "http://localhost:11434"; - - private String hoststub; - - public OllamaClient(String hoststub) { + + private final String hoststub; + + public OllamaClient(final String hoststub) { this.hoststub = hoststub; } public LinkedHashMap listOllamaModels() { - LinkedHashMap sortedMap = new LinkedHashMap<>(); + final LinkedHashMap sortedMap = new LinkedHashMap<>(); try { - String response = OpenAIClient.sendGetRequest(this.hoststub + "/api/tags"); - JSONObject responseObject = new JSONObject(response); - JSONArray models = responseObject.getJSONArray("models"); - - List> list = new ArrayList<>(); + final String response = OpenAIClient.sendGetRequest(this.hoststub + "/api/tags"); + final JSONObject responseObject = new JSONObject(response); + final JSONArray models = responseObject.getJSONArray("models"); + + final List> list = new ArrayList<>(); for (int i = 0; i < models.length(); i++) { - JSONObject model = models.getJSONObject(i); - String name = model.optString("name", ""); - long size = model.optLong("size", 0); - list.add(new AbstractMap.SimpleEntry(name, size)); + final JSONObject model = models.getJSONObject(i); + final String name = model.optString("name", ""); + final long size = model.optLong("size", 0); + list.add(new AbstractMap.SimpleEntry<>(name, size)); } // Sort the list in descending order based on the values list.sort((o1, o2) -> o2.getValue().compareTo(o1.getValue())); - + // Create a new LinkedHashMap and add the sorted entries - for (Map.Entry entry : list) { + for (final Map.Entry entry : list) { sortedMap.put(entry.getKey(), entry.getValue()); } } catch (JSONException | URISyntaxException | IOException e) { @@ -70,8 +70,8 @@ public LinkedHashMap listOllamaModels() { return sortedMap; } - public boolean ollamaModelExists(String name) { - JSONObject data = new JSONObject(); + public boolean ollamaModelExists(final String name) { + final JSONObject data = new JSONObject(); try { data.put("name", name); OpenAIClient.sendPostRequest(this.hoststub + "/api/show", data); @@ -81,37 +81,37 @@ public boolean ollamaModelExists(String name) { } } - public boolean pullOllamaModel(String name) { - JSONObject data = new JSONObject(); + public boolean pullOllamaModel(final String name) { + final JSONObject data = new JSONObject(); try { data.put("name", name); data.put("stream", false); - String response = OpenAIClient.sendPostRequest(this.hoststub + "/api/pull", data); + final String response = OpenAIClient.sendPostRequest(this.hoststub + "/api/pull", data); // this sends {"status": "success"} in case of success - JSONObject responseObject = new JSONObject(response); - String status = responseObject.optString("status", ""); + final JSONObject responseObject = new JSONObject(response); + final String status = responseObject.optString("status", ""); return status.equals("success"); } catch (JSONException | URISyntaxException | IOException e) { return false; } } - - public static void main(String[] args) { - OllamaClient oc = new OllamaClient(OLLAMA_API_HOST); - - LinkedHashMap models = oc.listOllamaModels(); + + public static void main(final String[] args) { + final OllamaClient oc = new OllamaClient(OLLAMA_API_HOST); + + final LinkedHashMap models = oc.listOllamaModels(); System.out.println(models.toString()); // check if model exists - String model = "phi3:3.8b"; + final String model = "phi3:3.8b"; if (oc.ollamaModelExists(model)) System.out.println("model " + model + " exists"); else System.out.println("model " + model + " does not exist"); - + // pull a model - boolean success = oc.pullOllamaModel(model); + final boolean success = oc.pullOllamaModel(model); System.out.println("pulled model + " + model + ": " + success); - + } } diff --git a/source/net/yacy/ai/OpenAIClient.java b/source/net/yacy/ai/OpenAIClient.java index d823426ecb..f068eadd2e 100644 --- a/source/net/yacy/ai/OpenAIClient.java +++ b/source/net/yacy/ai/OpenAIClient.java @@ -1,7 +1,7 @@ /** * OpenAIClient * Copyright 2024 by Michael Peter Christen - * First released 17.05.2024 at http://yacy.net + * First released 17.05.2024 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -38,31 +38,31 @@ public class OpenAIClient { private static String[] STOPTOKENS = new String[]{"[/INST]", "<|im_end|>", "<|end_of_turn|>", "<|eot_id|>", "<|end_header_id|>", "", "", "<|end|>"}; - private String hoststub; - - public OpenAIClient(String hoststub) { + private final String hoststub; + + public OpenAIClient(final String hoststub) { this.hoststub = hoststub; } // API Helper Methods - - public static String sendPostRequest(String endpoint, JSONObject data) throws IOException, URISyntaxException { - URL url = new URI(endpoint).toURL(); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + + public static String sendPostRequest(final String endpoint, final JSONObject data) throws IOException, URISyntaxException { + final URL url = new URI(endpoint).toURL(); + final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("POST"); conn.setRequestProperty("Content-Type", "application/json"); conn.setDoOutput(true); try (OutputStream os = conn.getOutputStream()) { - byte[] input = data.toString().getBytes("utf-8"); + final byte[] input = data.toString().getBytes("utf-8"); os.write(input, 0, input.length); } - int responseCode = conn.getResponseCode(); + final int responseCode = conn.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_OK) { try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) { - StringBuilder response = new StringBuilder(); + final StringBuilder response = new StringBuilder(); String responseLine; while ((responseLine = br.readLine()) != null) { response.append(responseLine.trim()); @@ -74,15 +74,15 @@ public static String sendPostRequest(String endpoint, JSONObject data) throws IO } } - public static String sendGetRequest(String endpoint) throws IOException, URISyntaxException { - URL url = new URI(endpoint).toURL(); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + public static String sendGetRequest(final String endpoint) throws IOException, URISyntaxException { + final URL url = new URI(endpoint).toURL(); + final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("GET"); - int responseCode = conn.getResponseCode(); + final int responseCode = conn.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_OK) { try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) { - StringBuilder response = new StringBuilder(); + final StringBuilder response = new StringBuilder(); String responseLine; while ((responseLine = br.readLine()) != null) { response.append(responseLine.trim()); @@ -93,14 +93,14 @@ public static String sendGetRequest(String endpoint) throws IOException, URISynt throw new IOException("Request failed with response code " + responseCode); } } - - // OpenAI chat client, works with llama.cpp and Ollama - - public String chat(String model, String prompt, int max_tokens) throws IOException { - JSONObject data = new JSONObject(); - JSONArray messages = new JSONArray(); - JSONObject systemPrompt = new JSONObject(true); - JSONObject userPrompt = new JSONObject(true); + + // OpenAI chat client, works with llama.cpp and Ollama + + public String chat(final String model, final String prompt, final int max_tokens) throws IOException { + final JSONObject data = new JSONObject(); + final JSONArray messages = new JSONArray(); + final JSONObject systemPrompt = new JSONObject(true); + final JSONObject userPrompt = new JSONObject(true); messages.put(systemPrompt); messages.put(userPrompt); try { @@ -114,52 +114,52 @@ public String chat(String model, String prompt, int max_tokens) throws IOExcepti data.put("messages", messages); data.put("stop", new JSONArray(STOPTOKENS)); data.put("stream", false); - String response = sendPostRequest(this.hoststub + "/v1/chat/completions", data); - JSONObject responseObject = new JSONObject(response); - JSONArray choices = responseObject.getJSONArray("choices"); - JSONObject choice = choices.getJSONObject(0); - JSONObject message = choice.getJSONObject("message"); - String content = message.optString("content", ""); + final String response = sendPostRequest(this.hoststub + "/v1/chat/completions", data); + final JSONObject responseObject = new JSONObject(response); + final JSONArray choices = responseObject.getJSONArray("choices"); + final JSONObject choice = choices.getJSONObject(0); + final JSONObject message = choice.getJSONObject("message"); + final String content = message.optString("content", ""); return content; } catch (JSONException | URISyntaxException e) { throw new IOException(e.getMessage()); } } - public static String[] stringsFromChat(String answer) { - int p = answer.indexOf('['); - int q = answer.indexOf(']'); + public static String[] stringsFromChat(final String answer) { + final int p = answer.indexOf('['); + final int q = answer.indexOf(']'); if (p < 0 || q < 0 || q < p) return new String[0]; try { - JSONArray a = new JSONArray(answer.substring(p, q + 1)); - String[] arr = new String[a.length()]; + final JSONArray a = new JSONArray(answer.substring(p, q + 1)); + final String[] arr = new String[a.length()]; for (int i = 0; i < a.length(); i++) arr[i] = a.getString(i); return arr; - } catch (JSONException e) { + } catch (final JSONException e) { return new String[0]; } } - - public static void main(String[] args) { - String model = "phi3:3.8b"; - OpenAIClient oaic = new OpenAIClient(OllamaClient.OLLAMA_API_HOST); + + public static void main(final String[] args) { + final String model = "phi3:3.8b"; + final OpenAIClient oaic = new OpenAIClient(OllamaClient.OLLAMA_API_HOST); // make chat completion with model String question = "Who invented the wheel?"; try { - String answer = oaic.chat(model, question, 80); + final String answer = oaic.chat(model, question, 80); System.out.println(answer); - } catch (IOException e) { + } catch (final IOException e) { e.printStackTrace(); } // try the json parser from chat results question = "Make a list of four names from Star Wars movies. Use a JSON Array."; try { - String[] a = stringsFromChat(oaic.chat(model, question, 80)); - for (String s: a) System.out.println(s); - } catch (IOException e) { + final String[] a = stringsFromChat(oaic.chat(model, question, 80)); + for (final String s: a) System.out.println(s); + } catch (final IOException e) { e.printStackTrace(); } } - + } diff --git a/source/net/yacy/cora/date/AbstractFormatter.java b/source/net/yacy/cora/date/AbstractFormatter.java index e2c8347204..d5d6d9fbbd 100644 --- a/source/net/yacy/cora/date/AbstractFormatter.java +++ b/source/net/yacy/cora/date/AbstractFormatter.java @@ -1,7 +1,7 @@ /** * AbstractFormatter * Copyright 2011 by Michael Peter Christen - * First released 2.1.2011 at http://yacy.net + * First released 2.1.2011 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ @@ -11,12 +11,12 @@ * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -62,7 +62,7 @@ public abstract class AbstractFormatter implements DateFormatter { @Override public abstract String format(); - private static final HashMap DATE_FORMAT_REGEXPS = new HashMap() { + private static final HashMap DATE_FORMAT_REGEXPS = new HashMap<>() { private static final long serialVersionUID = 1321140786174228717L; { put(Pattern.compile("^\\d{8}$"), new SimpleDateFormat("yyyyMMdd", Locale.US)); @@ -92,16 +92,16 @@ public abstract class AbstractFormatter implements DateFormatter { }}; @Override - public Date parse(String s) { + public Date parse(final String s) { return parseAny(s); } - public static Date parseAny(String s) { - for (Map.Entry ps: DATE_FORMAT_REGEXPS.entrySet()) { + public static Date parseAny(final String s) { + for (final Map.Entry ps: DATE_FORMAT_REGEXPS.entrySet()) { if (ps.getKey().matcher(s.toLowerCase()).matches()) { try { return ps.getValue().parse(s); - } catch (ParseException e) { + } catch (final ParseException e) { } } } diff --git a/source/net/yacy/cora/date/DateFormatter.java b/source/net/yacy/cora/date/DateFormatter.java index 261e0fa975..1163bd4ed3 100644 --- a/source/net/yacy/cora/date/DateFormatter.java +++ b/source/net/yacy/cora/date/DateFormatter.java @@ -1,7 +1,7 @@ /** * DateFormatter * Copyright 2011 by Michael Peter Christen - * First released 2.1.2011 at http://yacy.net + * First released 2.1.2011 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ @@ -11,12 +11,12 @@ * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -34,5 +34,5 @@ public interface DateFormatter { public Calendar parse(String s, int timezoneOffset) throws ParseException; public String format(final Date date); public String format(); - + } diff --git a/source/net/yacy/cora/date/GenericFormatter.java b/source/net/yacy/cora/date/GenericFormatter.java index 31a5ac67d5..09f0d16fbf 100644 --- a/source/net/yacy/cora/date/GenericFormatter.java +++ b/source/net/yacy/cora/date/GenericFormatter.java @@ -1,7 +1,7 @@ /** * GenericFormatter * Copyright 2011 by Michael Peter Christen - * First released 2.1.2011 at http://yacy.net + * First released 2.1.2011 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ @@ -50,7 +50,7 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter public static final String PATTERN_RFC1123_SHORT = "EEE, dd MMM yyyy"; public static final String PATTERN_ANSIC = "EEE MMM d HH:mm:ss yyyy"; public static final String PATTERN_SIMPLE = "yyyy/MM/dd HH:mm:ss"; - + /** * A regular expression matching the PATTERN_SIMPLE pattern (does not control * last day of month (30/31 or 28/29 for february). Can be used as a HTML5 input @@ -74,7 +74,7 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter public static final DateTimeFormatter FORMAT_SHORT_MINUTE = DateTimeFormatter .ofPattern(PATTERN_SHORT_MINUTE.replace("yyyy", "uuuu"), Locale.US).withZone(ZoneId.systemDefault()); - + /** * A thread-safe date formatter using the * {@link GenericFormatter#PATTERN_SHORT_SECOND} pattern with the US locale on @@ -257,17 +257,17 @@ public String format() { @Override public Calendar parse(final String timeString, final int timezoneOffset) throws ParseException { synchronized (this.dateFormat) { - Calendar cal = Calendar.getInstance(UTCtimeZone); + final Calendar cal = Calendar.getInstance(UTCtimeZone); cal.setTime(this.dateFormat.parse(timeString)); cal.add(Calendar.MINUTE, timezoneOffset); // add a correction; i.e. for UTC+1 -60 minutes is added to patch a time given in UTC+1 to the actual time at UTC return cal; } } - + /** * Like {@link #parseShortSecond(String)} using additional timezone information provided in an * offset String, like "+0100" for CET. - * @throws ParseException + * @throws ParseException */ public Calendar parse(final String timeString, final String UTCOffset) throws ParseException { if (timeString == null || timeString.isEmpty()) { return Calendar.getInstance(UTCtimeZone); } @@ -292,7 +292,7 @@ private static int UTCDiff(final String diffString) { final int om = NumberTools.parseIntDecSubstring(diffString, 3); return ((ahead) ? 1 : -1) * (oh * 60 + om); } - + /** * get the difference of this servers time zone to UTC/GMT in milliseconds * @return @@ -307,7 +307,7 @@ private static long UTCDiff() { return zoneOffsetHours + DSTOffsetHours; } } - + public static String UTCDiffString() { // we express the UTC Difference in 5 digits: // SHHMM @@ -330,12 +330,12 @@ public static String UTCDiffString() { } private final static DecimalFormat D2 = new DecimalFormat("00"); - + /** * Safely format the given time value using the given formatter. Fallback to * ISO-8601 representation or to raw time value without exception when the * format can not be applied. - * + * * @param time * a time value as millisecnods from Epoch (1970-01-01T00:00:00Z) * @param formatter @@ -355,11 +355,11 @@ public static String formatSafely(final long time, final DateTimeFormatter forma } return res; } - + /** * Safely format the given instant using the given formatter. Fallback to * ISO-8601 representation without exception when the format can not be applied. - * + * * @param instant * the instant to format * @param formatter @@ -384,7 +384,7 @@ public static String formatSafely(final Instant instant, final DateTimeFormatter return res; } - public static void main(String[] args) { + public static void main(final String[] args) { System.out.println(UTCDiffString()); } } diff --git a/source/net/yacy/cora/date/ISO8601Formatter.java b/source/net/yacy/cora/date/ISO8601Formatter.java index 7c74143f83..a16d80ca43 100644 --- a/source/net/yacy/cora/date/ISO8601Formatter.java +++ b/source/net/yacy/cora/date/ISO8601Formatter.java @@ -1,7 +1,7 @@ /** * ISO8601 * Copyright 2011 by Michael Peter Christen - * First released 2.1.2011 at http://yacy.net + * First released 2.1.2011 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ @@ -11,12 +11,12 @@ * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -36,25 +36,25 @@ public class ISO8601Formatter extends AbstractFormatter implements DateFormatter /** pattern for a W3C datetime variant of a non-localized ISO8601 date */ private static final String PATTERN_ISO8601 = "yyyy-MM-dd'T'HH:mm:ss'Z'"; - + /** Date formatter/non-sloppy parser for W3C datetime (ISO8601) in GMT/UTC */ private static final SimpleDateFormat FORMAT_ISO8601 = new SimpleDateFormat(PATTERN_ISO8601, Locale.US); - + static { FORMAT_ISO8601.setTimeZone(AbstractFormatter.UTCtimeZone); } - + public static final ISO8601Formatter FORMATTER = new ISO8601Formatter(); public ISO8601Formatter() { - last_time = 0; - last_format = ""; + this.last_time = 0; + this.last_format = ""; } - + /** * Parse dates as defined in http://www.w3.org/TR/NOTE-datetime. * This format (also specified in ISO8601) allows different "precisions". - * The following lower precision versions for the complete date + * The following lower precision versions for the complete date * "2007-12-19T10:20:30.567+0300" are allowed:
* "2007"
* "2007-12"
@@ -66,7 +66,7 @@ public ISO8601Formatter() { * Parsing is done in a fuzzy way. If there is an illegal character somewhere in * the String, the date parsed so far will be returned, e.g. the input * "2007-12-19FOO" would return a date that represents "2007-12-19". - * + * * @param s * @return * @throws ParseException @@ -85,17 +85,17 @@ public Calendar parse(String s, final int timezoneOffset) throws ParseException int p = s.indexOf(';'); if (p >= 0) s = s.substring(0, p); // a semicolon may be used to separate two dates from each other; then we take the first p = s.indexOf(','); if (p >= 0) s = s.substring(0, p); // a comma may be used to separate two dates from each other; then we take the first while (!s.isEmpty() && s.endsWith("?")) s = s.substring(0, s.length() - 1); // sometimes used if write is not sure about date - + // no go for exact parsing final Calendar cal = Calendar.getInstance(AbstractFormatter.UTCtimeZone, Locale.US); cal.clear(); - + // split 2007-12-19T10:20:30.789+0500 into its parts // correct: yyyy['-'MM['-'dd['T'HH':'MM[':'ss['.'SSS]]('Z'|ZZZZZ)]]] final StringTokenizer t = new StringTokenizer(s, "-T:.Z+", true); if (s == null || t.countTokens() == 0) throw new ParseException("parseISO8601: Cannot parse '" + s + "'", 0); - + try { // year cal.set(Calendar.YEAR, Integer.parseInt(t.nextToken())); @@ -134,7 +134,7 @@ public Calendar parse(String s, final int timezoneOffset) throws ParseException token = t.nextToken(); } } - + // check for TZ data int offset; if (token.equals("Z")) { @@ -158,13 +158,11 @@ public Calendar parse(String s, final int timezoneOffset) throws ParseException cal.set(Calendar.SECOND, sec); cal.set(Calendar.MILLISECOND, msec); } - } catch (final NoSuchElementException e) { - // ignore this as it is perfectly fine to have non-complete date in this format } catch (final Exception e) { // catch all Exceptions and return what we parsed so far //serverLog.logInfo("SERVER", "parseISO8601: DATE ERROR with: '" + s + "' got so far: '" + cal.toString()); } - + // in case we couldn't even parse a year if (!cal.isSet(Calendar.YEAR)) throw new ParseException("parseISO8601: Cannot parse '" + s + "'", 0); @@ -176,29 +174,29 @@ public Calendar parse(String s, final int timezoneOffset) throws ParseException * Creates a String representation of a Date using the format defined * in ISO8601/W3C datetime * The result will be in UTC/GMT, e.g. "2007-12-19T10:20:30Z". - * + * * @param date The Date instance to transform. * @return A fixed width (20 chars) ISO8601 date String. */ @Override public final String format(final Date date) { if (date == null) return ""; - if (Math.abs(date.getTime() - last_time) < 1000) return last_format; + if (Math.abs(date.getTime() - this.last_time) < 1000) return this.last_format; synchronized (FORMAT_ISO8601) { - last_format = FORMAT_ISO8601.format(date); - last_time = date.getTime(); + this.last_format = FORMAT_ISO8601.format(date); + this.last_time = date.getTime(); } - return last_format; + return this.last_format; } @Override public final String format() { - long time = System.currentTimeMillis(); - if (Math.abs(time - last_time) < 1000) return last_format; + final long time = System.currentTimeMillis(); + if (Math.abs(time - this.last_time) < 1000) return this.last_format; synchronized (FORMAT_ISO8601) { - last_format = FORMAT_ISO8601.format(new Date(time)); - last_time = time; + this.last_format = FORMAT_ISO8601.format(new Date(time)); + this.last_time = time; } - return last_format; + return this.last_format; } - + } diff --git a/source/net/yacy/cora/date/MicroDate.java b/source/net/yacy/cora/date/MicroDate.java index 8563b6e46d..9c4624c008 100644 --- a/source/net/yacy/cora/date/MicroDate.java +++ b/source/net/yacy/cora/date/MicroDate.java @@ -1,7 +1,7 @@ /** * MicroDate * Copyright 2008 by Michael Peter Christen - * First released 3.7.2008 at http://yacy.net + * First released 3.7.2008 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -29,11 +29,11 @@ public class MicroDate { private static final long hour = 3600000L; // milliseconds of a hour private static final long day = 86400000L; // milliseconds of a day - + public static int microDateDays(final Date modified) { return microDateDays(modified.getTime()); } - + public static int microDateDays(final long modified) { // this calculates a virtual age from a given date // the purpose is to have an age in days of a given modified date @@ -42,20 +42,20 @@ public static int microDateDays(final long modified) { // we take mod 64**3 = 262144, this is the mask of the storage return (int) ((modified / day) % 262144L); } - + public static int microDateHoursInt(final long time) { return (int) ((time / hour) % 262144L); } - + public static int microDateHoursAge(final String mdhs) { return microDateHoursInt(System.currentTimeMillis()) - (int) Base64Order.enhancedCoder.decodeLong(mdhs); } - + public static long reverseMicroDateDays(final long microDateDays) { return Math.min(System.currentTimeMillis(), microDateDays * day); } - + public static void main(final String[] args) { - + } } diff --git a/source/net/yacy/cora/document/WordCache.java b/source/net/yacy/cora/document/WordCache.java index 968a30abf5..f43672374a 100644 --- a/source/net/yacy/cora/document/WordCache.java +++ b/source/net/yacy/cora/document/WordCache.java @@ -1,7 +1,7 @@ /** * WordCache * Copyright 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * first published 01.10.2009 on http://yacy.net + * first published 01.10.2009 on https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -50,11 +50,11 @@ public class WordCache { private final static ConcurrentLog log = new ConcurrentLog(WordCache.class.getName()); - + // common word cache private static final int commonWordsMaxSize = 20000; // maximum size of common word cache private static final int commonWordsMinLength = 5; // words must have that length at minimum - private static OrderedScoreMap commonWords = new OrderedScoreMap(StringBuilderComparator.CASE_INSENSITIVE_ORDER); + private static OrderedScoreMap commonWords = new OrderedScoreMap<>(StringBuilderComparator.CASE_INSENSITIVE_ORDER); // dictionaries private final File dictionaryPath; @@ -67,8 +67,8 @@ public static class Dictionary { public Dictionary(final File file) throws IOException { - this.dict = new TreeSet(StringBuilderComparator.CASE_INSENSITIVE_ORDER); - this.tcid = new TreeSet(StringBuilderComparator.CASE_INSENSITIVE_ORDER); + this.dict = new TreeSet<>(StringBuilderComparator.CASE_INSENSITIVE_ORDER); + this.tcid = new TreeSet<>(StringBuilderComparator.CASE_INSENSITIVE_ORDER); InputStream is = new FileInputStream(file); if (file.getName().endsWith(".gz")) { @@ -103,7 +103,7 @@ public Dictionary(final File file) throws IOException { * @return set that contains all words that start or end with the input value */ public Set recommend(StringBuilder string) { - final Set ret = new HashSet(); + final Set ret = new HashSet<>(); SortedSet t = this.dict.tailSet(string); for (final StringBuilder r: t) { if (StringBuilderComparator.CASE_INSENSITIVE_ORDER.startsWith(r, string) && r.length() > string.length()) { @@ -201,7 +201,7 @@ public Map getDictionaries() { */ public WordCache(final File dictionaryPath) { this.dictionaryPath = dictionaryPath; - this.dictionaries = new ConcurrentHashMap(); + this.dictionaries = new ConcurrentHashMap<>(); reload(); } @@ -222,8 +222,8 @@ public static void learn(final StringBuilder word) { } } - public static void learn(Collection wordset) { - for (String s: wordset) { + public static void learn(final Collection wordset) { + for (final String s: wordset) { learn(new StringBuilder(s)); } } @@ -239,7 +239,7 @@ public void reload() { for (final String f: files) { if (f.endsWith(".words")) { try { - Dictionary dict = new Dictionary(new File(this.dictionaryPath, f)); + final Dictionary dict = new Dictionary(new File(this.dictionaryPath, f)); this.dictionaries.put(f.substring(0, f.length() - 6), dict); } catch (final IOException e) { log.warn(e); @@ -262,9 +262,9 @@ private static StringBuilder reverse(final StringBuilder s) { * @param string input value that is used to match recommendations * @return set that contains all words that start or end with the input value */ - public Set recommend(StringBuilder string) { - Set ret = new HashSet(); - for (Dictionary dict: this.dictionaries.values()) { + public Set recommend(final StringBuilder string) { + final Set ret = new HashSet<>(); + for (final Dictionary dict: this.dictionaries.values()) { ret.addAll(dict.recommend(string)); } final SortedMap u = commonWords.tailMap(string); @@ -290,7 +290,7 @@ public Set recommend(StringBuilder string) { * @return true if the library contains the word */ public boolean contains(final StringBuilder s) { - for (Dictionary dict: this.dictionaries.values()) { + for (final Dictionary dict: this.dictionaries.values()) { if (dict.contains(s)) { return true; } @@ -305,8 +305,8 @@ public boolean contains(final StringBuilder s) { * @param string the given word * @return true if the library supports the word */ - public boolean supports(StringBuilder string) { - for (Dictionary dict: this.dictionaries.values()) { + public boolean supports(final StringBuilder string) { + for (final Dictionary dict: this.dictionaries.values()) { if (dict.supports(string)) { return true; } @@ -320,7 +320,7 @@ public boolean supports(StringBuilder string) { */ public int size() { int size = 0; - for (Dictionary dict: this.dictionaries.values()) { + for (final Dictionary dict: this.dictionaries.values()) { size += dict.size(); } return size; @@ -329,7 +329,7 @@ public int size() { public static int sizeCommonWords() { return commonWords.size(); } - + public static void clearCommonWords() { commonWords.clear(); } @@ -343,7 +343,7 @@ public static void clearCommonWords() { * @return */ public boolean isRelevant(final int minimumWords) { - for (Dictionary dict: this.dictionaries.values()) { + for (final Dictionary dict: this.dictionaries.values()) { if (dict.isRelevant(minimumWords)) { return true; } diff --git a/source/net/yacy/cora/document/analysis/Classification.java b/source/net/yacy/cora/document/analysis/Classification.java index e04f849945..d5b9f326a6 100644 --- a/source/net/yacy/cora/document/analysis/Classification.java +++ b/source/net/yacy/cora/document/analysis/Classification.java @@ -1,7 +1,7 @@ /** * Classification.java * Copyright 2009 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 09.07.2009 at http://yacy.net + * First released 09.07.2009 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -34,13 +34,13 @@ public class Classification { - private static final Set textExtSet = new HashSet(); - private static final Set mediaExtSet = new HashSet(); - private static final Set imageExtSet = new HashSet(); - private static final Set audioExtSet = new HashSet(); - private static final Set videoExtSet = new HashSet(); - private static final Set appsExtSet = new HashSet(); - private static final Set ctrlExtSet = new HashSet(); + private static final Set textExtSet = new HashSet<>(); + private static final Set mediaExtSet = new HashSet<>(); + private static final Set imageExtSet = new HashSet<>(); + private static final Set audioExtSet = new HashSet<>(); + private static final Set videoExtSet = new HashSet<>(); + private static final Set appsExtSet = new HashSet<>(); + private static final Set ctrlExtSet = new HashSet<>(); public enum ContentDomain { @@ -54,7 +54,7 @@ public enum ContentDomain { private final int code; - ContentDomain(int code) { + ContentDomain(final int code) { this.code = code; } @@ -104,17 +104,17 @@ public String toString() { addSet(mediaExtSet, apps + "," + audio + "," + video + "," + image); // all media formats } - private static void addSet(Set set, final String extString) { + private static void addSet(final Set set, final String extString) { if ((extString == null) || (extString.isEmpty())) return; - for (String s: CommonPattern.COMMA.split(extString, 0)) set.add(s.toLowerCase(Locale.ROOT).trim()); + for (final String s: CommonPattern.COMMA.split(extString, 0)) set.add(s.toLowerCase(Locale.ROOT).trim()); } - public static boolean isTextExtension(String textExt) { + public static boolean isTextExtension(final String textExt) { if (textExt == null) return false; return textExtSet.contains(textExt.trim().toLowerCase(Locale.ROOT)); } - public static boolean isMediaExtension(String mediaExt) { + public static boolean isMediaExtension(final String mediaExt) { if (mediaExt == null) return false; return mediaExtSet.contains(mediaExt.trim().toLowerCase(Locale.ROOT)); } @@ -200,9 +200,9 @@ public static void init(final File mimeFile) { if (mimeTableInputStream != null) try { mimeTableInputStream.close(); } catch (final Exception e1) {} } } - for (Entry entry: mimeTable.entrySet()) { - String ext = (String) entry.getKey(); - String mime = (String) entry.getValue(); + for (final Entry entry: mimeTable.entrySet()) { + final String ext = (String) entry.getKey(); + final String mime = (String) entry.getValue(); if (mime.startsWith("text/")) textExtSet.add(ext.toLowerCase(Locale.ROOT)); if (mime.startsWith("audio/")) audioExtSet.add(ext.toLowerCase(Locale.ROOT)); if (mime.startsWith("video/")) videoExtSet.add(ext.toLowerCase(Locale.ROOT)); diff --git a/source/net/yacy/cora/document/encoding/ASCII.java b/source/net/yacy/cora/document/encoding/ASCII.java index d4c71fa39e..af401dbf6b 100644 --- a/source/net/yacy/cora/document/encoding/ASCII.java +++ b/source/net/yacy/cora/document/encoding/ASCII.java @@ -1,7 +1,7 @@ /** * ASCIIComparator * Copyright 2010 by Michael Peter Christen - * First released 25.2.2011 at http://yacy.net + * First released 25.2.2011 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ @@ -42,7 +42,7 @@ public class ASCII implements Comparator { public boolean insensitive; - public ASCII(boolean insensitive) { + public ASCII(final boolean insensitive) { this.insensitive = insensitive; } @@ -52,13 +52,13 @@ public Object clone() { } @Override - public int compare(String s0, String s1) { + public int compare(final String s0, final String s1) { if (s0 == null && s1 == null) return 0; if (s0 == null) return -1; if (s1 == null) return 1; int i = 0; - int l0 = s0.length(), l1 = s1.length(); - int lm = Math.min(l0, l1); + final int l0 = s0.length(), l1 = s1.length(); + final int lm = Math.min(l0, l1); char c0, c1; while (i < lm) { c0 = s0.charAt(i); @@ -74,13 +74,13 @@ public int compare(String s0, String s1) { return 0; } - public boolean equals(String s0, String s1) { + public boolean equals(final String s0, final String s1) { if (s0 == null && s1 == null) return true; if (s0 == null) return false; if (s1 == null) return false; int i = 0; - int l0 = s0.length(), l1 = s1.length(); - int lm = Math.min(l0, l1); + final int l0 = s0.length(), l1 = s1.length(); + final int lm = Math.min(l0, l1); char c0, c1; while (i < lm) { c0 = s0.charAt(i); @@ -95,7 +95,7 @@ public boolean equals(String s0, String s1) { } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { return (obj == this); } @@ -112,7 +112,7 @@ public final static String String(final byte[] bytes) { } public final static String String(final byte[] bytes, final int offset, final int length) { - int l = Math.min(length, bytes.length - offset); + final int l = Math.min(length, bytes.length - offset); final char[] c = new char[l]; for (int i = 0; i < l; ++ i) { if (bytes[i + offset] < 0) throw new IllegalArgumentException(); @@ -124,7 +124,7 @@ public final static String String(final byte[] bytes, final int offset, final in public final static byte[] getBytes(final String s) { assert s != null; //assert s.length() < 3 || s.charAt(2) != '@'; - int count = s.length(); + final int count = s.length(); final byte[] b = new byte[count]; for (int i = 0; i < count; i++) { b[i] = (byte) s.charAt(i); @@ -135,7 +135,7 @@ public final static byte[] getBytes(final String s) { public final static byte[] getBytes(final String s, final int beginIndex, final int endIndex) { assert s != null; //assert s.length() < 3 || s.charAt(2) != '@'; - int count = endIndex - beginIndex; + final int count = endIndex - beginIndex; final byte[] b = new byte[count]; for (int i = 0; i < count; i++) { b[i] = (byte) s.charAt(i + beginIndex); diff --git a/source/net/yacy/cora/document/encoding/UTF8.java b/source/net/yacy/cora/document/encoding/UTF8.java index 5a871cfd6f..8b500dd0cd 100644 --- a/source/net/yacy/cora/document/encoding/UTF8.java +++ b/source/net/yacy/cora/document/encoding/UTF8.java @@ -1,7 +1,7 @@ /** * UTF8 * Copyright 2011 by Michael Peter Christen - * First released 25.2.2011 at http://yacy.net + * First released 25.2.2011 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ @@ -50,12 +50,12 @@ public class UTF8 implements Comparator { public boolean insensitive; - public UTF8(boolean insensitive) { + public UTF8(final boolean insensitive) { this.insensitive = insensitive; } @Override - public int compare(String o0, String o1) { + public int compare(final String o0, final String o1) { final int l0 = o0.length(); final int l1 = o1.length(); final int ml = Math.min(l0, l1); @@ -96,7 +96,7 @@ private boolean equals(final String o0, final String o1, final int l) { } return true; } - + public final static StringBody StringBody(final byte[] b) { return StringBody(UTF8.String(b)); } @@ -270,12 +270,12 @@ public static String decodeURL(final String s) { return (needToChange? sb.toString() : s); } - + /** * Utility wrapper around the standard JDK {@link URLEncoder#encode(String)} * function, using the UTF-8 character set and catching * {@link UnsupportedEncodingException} exception. - * + * * @param str a string to encode. Must not be null. * @return str encoded in application/x-www-form-urlencoded format */ diff --git a/source/net/yacy/cora/document/feed/Channel.java b/source/net/yacy/cora/document/feed/Channel.java index 2200ae3ff5..2ba471915d 100644 --- a/source/net/yacy/cora/document/feed/Channel.java +++ b/source/net/yacy/cora/document/feed/Channel.java @@ -1,7 +1,7 @@ /** * Channel * Copyright 2010 by Michael Peter Christen - * First released 10.5.2010 at http://yacy.net + * First released 10.5.2010 at https//yacy.net * * $LastChangedDate$ * $LastChangedRevision$ @@ -13,12 +13,12 @@ * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -30,18 +30,18 @@ public interface Channel extends Iterable { public void setTitle(String title); - + public void setLink(String link); - + public void setDescription(String description); - + public void setImageURL(String imageUrl); - + public void setTotalResults(String totalResults); - + public void setStartIndex(String startIndex); - + public void setItemsPerPage(String itemsPerPage); - + public void setSearchTerms(String searchTerms); } diff --git a/source/net/yacy/cora/document/feed/Hit.java b/source/net/yacy/cora/document/feed/Hit.java index 3cb61ab711..06386fc8a2 100644 --- a/source/net/yacy/cora/document/feed/Hit.java +++ b/source/net/yacy/cora/document/feed/Hit.java @@ -1,7 +1,7 @@ /** * Hit * Copyright 2010 by Michael Peter Christen - * First released 10.5.2010 at http://yacy.net + * First released 10.5.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ @@ -13,12 +13,12 @@ * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -32,68 +32,68 @@ public interface Hit { public void setAuthor(String title); - + public void setCopyright(String title); - + public void setCategory(String title); - + public void setTitle(String title); - + public void setLink(String link); - + public void setReferrer(String title); - + public void setLanguage(String title); - + public void setDescription(String description); - + public void setPubDate(Date pubdate); - + public void setGuid(String guid); - + public void setDocs(String docs); public void setSubject(String[] tags); - + public void setSize(long size); - + /** * @param enclosure an URL describing a media object that is attached to a feed item */ public void setEnclosure(String enclosure); - + public String getAuthor(); - + public String getCopyright(); - + public String getCategory(); - + public String getTitle(); - + public String getLink(); - + public String getReferrer(); - + public String getLanguage(); - + public List getDescriptions(); - + public Date getPubDate(); - + public String getGuid(); - + public String getDocs(); // A URL that points to the documentation for the format used in the RSS file. public String[] getSubject(); - + public long getSize(); - + /** * @return an URL describing a media object that is attached to a feed item */ public String getEnclosure(); public double getLon(); - + public double getLat(); } diff --git a/source/net/yacy/cora/document/feed/RSSFeed.java b/source/net/yacy/cora/document/feed/RSSFeed.java index b9b95be8cf..ff59386c2a 100644 --- a/source/net/yacy/cora/document/feed/RSSFeed.java +++ b/source/net/yacy/cora/document/feed/RSSFeed.java @@ -1,7 +1,7 @@ /** * RSSFeed * Copyright 2007 by Michael Peter Christen - * First released 16.7.2007 at http://yacy.net + * First released 16.7.2007 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -36,22 +36,22 @@ public class RSSFeed implements Iterable { public static final int DEFAULT_MAXSIZE = 10000; // class variables - + /** Single required element see http://www.rssboard.org/rss-profile#element-channel */ private RSSMessage channel = null; - + /** A guid:Item map */ private final Map messages; private final int maxsize; - + /** Set to true when maxsize messages limit has been exceeded and exceeding messages have been discarded */ private boolean maxSizeExceeded; - + @Override public String toString() { - StringBuilder sb = new StringBuilder(); + final StringBuilder sb = new StringBuilder(); sb.append("\n"); sb.append("0\n"); sb.append("" + this.size() + "\n"); sb.append("" + this.size() + "\n"); - for (RSSMessage item: messages.values()) { + for (final RSSMessage item: this.messages.values()) { sb.append(item.toString()); } sb.append("\n"); sb.append("\n"); return sb.toString(); } - + public RSSFeed(final int maxsize) { this.messages = Collections.synchronizedMap(new LinkedHashMap()); this.channel = null; @@ -83,11 +83,11 @@ public RSSFeed(final int maxsize) { * @param links * @param source */ - public RSSFeed(Set links, String source) { + public RSSFeed(final Set links, final String source) { this(Integer.MAX_VALUE); String u; RSSMessage message; - for (MultiProtocolURL uri: links) { + for (final MultiProtocolURL uri: links) { u = uri.toNormalform(true); message = new RSSMessage(u, "", u); message.setAuthor(source); @@ -112,8 +112,8 @@ public RSSMessage getChannel() { } public Set getLinks() { - Set links = new HashSet(); - for (RSSMessage message: this.messages.values()) { + final Set links = new HashSet<>(); + for (final RSSMessage message: this.messages.values()) { try {links.add(new MultiProtocolURL(message.getLink()));} catch (final MalformedURLException e) {} } return links; @@ -141,7 +141,7 @@ public boolean isEmpty() { public int size() { return this.messages.size(); } - + /** * @return true when maxsize messages limit has been exceeded and exceeding messages have been discarded */ diff --git a/source/net/yacy/cora/document/feed/RSSMessage.java b/source/net/yacy/cora/document/feed/RSSMessage.java index 8906d103ba..b2e55a2102 100644 --- a/source/net/yacy/cora/document/feed/RSSMessage.java +++ b/source/net/yacy/cora/document/feed/RSSMessage.java @@ -1,7 +1,7 @@ /** * RSSMessage * Copyright 2007 by Michael Peter Christen - * First released 16.7.2007 at http://yacy.net + * First released 16.7.2007 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ @@ -58,53 +58,53 @@ public static enum Token { /** Human-readable title for an entry or a feed. */ title(new String[]{"title","atom:title","rss:title",DublinCore.Title.getURIref()}), - + /** Reference from an entry or feed to a Web resource URL */ link(new String[]{"link","atom:link","rss:link"}), - + /** Human-readable description or subtitle for an entry or a feed. */ description(new String[]{"description","subtitle","atom:subtitle","rss:description", DublinCore.Description.getURIref()}), - + /** The publication date for content in a feed or for an antry. */ pubDate(new String[]{"pubDate","lastBuildDate","updated","rss:lastBuildDate","rss:updated"}), - + /** Copyright notice for content in the channel. */ copyright(new String[]{"copyright","publisher",DublinCore.Publisher.getURIref()}), - + /** The author of an item (Email address) */ author(new String[]{"author","creator",DublinCore.Creator.getURIref()}), - + subject(new String[]{"subject",DublinCore.Subject.getURIref()}), - + /** One or more categories a channel or item belongs to */ category(new String[]{"category"}), - + referrer(new String[]{"referrer","referer"}), - + /** The language the channel is written in. */ language(new String[]{"language",DublinCore.Language.getURIref()}), - + /** A string that uniquely identifies an item (RSS 2.0) */ guid(new String[]{"guid"}), - + /** URL describing a media object that is attached to a feed item */ enclosure(new String[]{"enclosure"}), - + /** Time To Live : number of minutes that indicates how long a channel (RSS 2.0) can be cached before refreshing from the source. */ ttl(new String[]{"ttl"}), - + /** URL to the documentation for the format used in the RSS file (for example http://www.rssboard.org/rss-specification) */ docs(new String[]{"docs"}), - + size(new String[]{"size", "length", YaCyMetadata.size.getURIref()}), lon(new String[]{Geo.Long.getURIref(), "geo:lon"}), // include possible misspelling geo:lon (instead of geo:long) lat(new String[]{Geo.Lat.getURIref()}); //point("gml:pos,georss:point,coordinates"); - + private Set keys; private Token(final String[] keylist) { - this.keys = new HashSet(); + this.keys = new HashSet<>(); this.keys.addAll(Arrays.asList(keylist)); this.keys.add(this.name()); } @@ -127,23 +127,23 @@ public String toString() { return this.keys.size() == 0 ? "" : this.keys.iterator().next(); } } - - private static Map tokenNick2Token = new HashMap(); + + private static Map tokenNick2Token = new HashMap<>(); static { - for (Token t: Token.values()) { - for (String nick: t.keys) tokenNick2Token.put(nick, t); + for (final Token t: Token.values()) { + for (final String nick: t.keys) tokenNick2Token.put(nick, t); } } - public static Token valueOfNick(String nick) { + public static Token valueOfNick(final String nick) { return tokenNick2Token.get(nick); } - + private static String artificialGuidPrefix = "c0_"; private static String calculatedGuidPrefix = "c1_"; public static final RSSMessage POISON = new RSSMessage("", "", ""); - public static final HashSet tags = new HashSet(); + public static final HashSet tags = new HashSet<>(); static { for (final Token token: Token.values()) { tags.addAll(token.keys()); @@ -153,7 +153,7 @@ public static Token valueOfNick(String nick) { private final Map map; public RSSMessage(final String title, final String description, final String link) { - this.map = new HashMap(); + this.map = new HashMap<>(); if (title.length() > 0) this.map.put(Token.title.name(), title); if (description.length() > 0) this.map.put(Token.description.name(), description); if (link.length() > 0) this.map.put(Token.link.name(), link); @@ -162,7 +162,7 @@ public RSSMessage(final String title, final String description, final String lin } public RSSMessage(final String title, final String description, final MultiProtocolURL link, final String guid) { - this.map = new HashMap(); + this.map = new HashMap<>(); if (title.length() > 0) this.map.put(Token.title.name(), title); if (description.length() > 0) this.map.put(Token.description.name(), description); this.map.put(Token.link.name(), link.toNormalform(true)); @@ -173,9 +173,9 @@ public RSSMessage(final String title, final String description, final MultiProto } public RSSMessage() { - this.map = new HashMap(); + this.map = new HashMap<>(); } - + public void setValue(final Token token, final String value) { if (value.length() > 0) { this.map.put(token.name(), value); @@ -214,8 +214,8 @@ public int compare(final RSSMessage o1, final RSSMessage o2) { @Override public List getDescriptions() { - List ds = new ArrayList(); - String d = Token.description.valueFrom(this.map, ""); + final List ds = new ArrayList<>(); + final String d = Token.description.valueFrom(this.map, ""); if (d.length() > 0) ds.add(d); return ds; } @@ -285,12 +285,12 @@ public String getGuid() { } return guid; } - + @Override public String getEnclosure() { return Token.enclosure.valueFrom(this.map, ""); } - + public String getTTL() { return Token.ttl.valueFrom(this.map, ""); } @@ -330,9 +330,9 @@ public double getLat() { public String toString() { return this.toString(true); } - - public String toString(boolean withItemTag) { - StringBuilder sb = new StringBuilder(); + + public String toString(final boolean withItemTag) { + final StringBuilder sb = new StringBuilder(); if (withItemTag) sb.append("\n"); if (this.map.containsKey(Token.title.name())) sb.append("").append(this.map.get(Token.title.name())).append("\n"); if (this.map.containsKey(Token.link.name())) sb.append("").append(this.map.get(Token.link.name())).append("\n"); @@ -384,12 +384,12 @@ public void setDocs(final String docs) { public void setGuid(final String guid) { setValue(Token.guid, guid); } - + @Override public void setEnclosure(final String enclosure) { setValue(Token.enclosure, enclosure); } - + @Override public void setLanguage(final String language) { setValue(Token.language, language); diff --git a/source/net/yacy/cora/document/feed/RSSReader.java b/source/net/yacy/cora/document/feed/RSSReader.java index 49f02f5ecf..2fb0126d0e 100644 --- a/source/net/yacy/cora/document/feed/RSSReader.java +++ b/source/net/yacy/cora/document/feed/RSSReader.java @@ -1,7 +1,7 @@ /** * RSSReader * Copyright 2007 by Michael Peter Christen - * First released 16.7.2007 at http://yacy.net + * First released 16.7.2007 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -50,7 +50,7 @@ public class RSSReader extends DefaultHandler { private boolean parsingChannel, parsingItem; private final RSSFeed theChannel; private Type type; - + /** When a parsing limit on instance construction has been exceeded */ private boolean maxBytesExceeded; @@ -66,7 +66,7 @@ private RSSReader(final int maxsize) { this.maxBytesExceeded = false; } - private static final ThreadLocal tlSax = new ThreadLocal(); + private static final ThreadLocal tlSax = new ThreadLocal<>(); private static SAXParser getParser() throws SAXException { SAXParser parser = tlSax.get(); if (parser == null) { @@ -98,16 +98,16 @@ public InputSource resolveEntity(final String arg0, final String arg1) throw new IOException (e.getMessage()); } } - + public RSSReader(final int maxsize, final long maxBytes, InputStream stream) throws IOException { this(maxsize); - + if (!(stream instanceof ByteArrayInputStream) && !(stream instanceof BufferedInputStream)) { stream = new BufferedInputStream(stream); } - - StrictLimitInputStream limitedSource = new StrictLimitInputStream(stream, maxBytes); - + + final StrictLimitInputStream limitedSource = new StrictLimitInputStream(stream, maxBytes); + try { final SAXParser saxParser = getParser(); // do not look at external dtd - see: http://www.ibm.com/developerworks/xml/library/x-tipcfsx/index.html @@ -121,7 +121,7 @@ public InputSource resolveEntity(final String arg0, final String arg1) saxParser.parse(limitedSource, this); } catch (final SAXException e) { throw new IOException (e.getMessage()); - } catch(StreamLimitException e) { + } catch(final StreamLimitException e) { this.maxBytesExceeded = true; } } @@ -194,7 +194,7 @@ public void startElement(final String uri, final String name, final String tag, if(StringUtils.isNotBlank(url)) { this.item.setEnclosure(url); } - } + } } } else if ("rss".equals(tag)) { this.type = Type.rss; @@ -236,7 +236,7 @@ public void characters(final char ch[], final int start, final int length) { public RSSFeed getFeed() { return this.theChannel; } - + /** * @return true when a parsing limit on instance construction has been exceeded */ diff --git a/source/net/yacy/cora/document/id/AnchorURL.java b/source/net/yacy/cora/document/id/AnchorURL.java index f63fc06191..820f9939ee 100644 --- a/source/net/yacy/cora/document/id/AnchorURL.java +++ b/source/net/yacy/cora/document/id/AnchorURL.java @@ -1,18 +1,18 @@ /** * Anchor * Copyright 2013 by Michael Peter Christen - * first published 15.09.2013 on http://yacy.net + * first published 15.09.2013 on https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -40,7 +40,7 @@ public AnchorURL(final String url) throws MalformedURLException { this.imageURL = null; this.imageAlt = null; } - + public AnchorURL(final AnchorURL url) { super(url, url.hash()); this.textBody = url.textBody; @@ -76,7 +76,7 @@ public AnchorURL(final String protocol, final String host, final int port, final this.imageURL = null; this.imageAlt = null; } - + public AnchorURL(final String protocol, final String host, final int port, final String path, final DigestURL imageURL, final String imageAlt) throws MalformedURLException { super(protocol, host, port, path); this.textBody = ""; @@ -104,42 +104,42 @@ public static AnchorURL newAnchor(final DigestURL baseURL, String relPath) throw } public String getNameProperty() { - return nameProperty; + return this.nameProperty; } - public void setNameProperty(String name) { + public void setNameProperty(final String name) { this.nameProperty = name; } public String getTextProperty() { - return textBody; + return this.textBody; } - public void setTextProperty(String text) { + public void setTextProperty(final String text) { this.textBody = text; } public String getRelProperty() { - return relProperty; + return this.relProperty; } - public void setRelProperty(String rel) { + public void setRelProperty(final String rel) { this.relProperty = rel; } public DigestURL getImageURL() { - return imageURL; + return this.imageURL; } - public void setImageURL(DigestURL imageURL) { + public void setImageURL(final DigestURL imageURL) { this.imageURL = imageURL; } public String getImageAlt() { - return imageAlt; + return this.imageAlt; } - public void setImageAlt(String imageAlt) { + public void setImageAlt(final String imageAlt) { this.imageAlt = imageAlt; } @@ -156,16 +156,16 @@ public Properties getAll() { tagopts.setProperty("rel", this.relProperty); return tagopts; } - + public boolean attachedNofollow() { return this.relProperty.indexOf("nofollow") >= 0; } - + public String toHTML() { return " 0 ? (" name=\"" + this.nameProperty + "\"") : "") + (this.relProperty.length() > 0 ? (" rel=\"" + this.relProperty + "\"") : "") + ">" + this.textBody + ""; } - + } diff --git a/source/net/yacy/cora/document/id/DigestURL.java b/source/net/yacy/cora/document/id/DigestURL.java index 93b7f00a25..f6ccdcbd48 100644 --- a/source/net/yacy/cora/document/id/DigestURL.java +++ b/source/net/yacy/cora/document/id/DigestURL.java @@ -1,18 +1,18 @@ /** * DigestURL * Copyright 2006 by Michael Peter Christen - * first published 13.07.2006 on http://yacy.net + * first published 13.07.2006 on https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -46,7 +46,7 @@ public class DigestURL extends MultiProtocolURL implements Serializable { public static final DigestURL POISON = new DigestURL(); // poison pill for concurrent link generators - + private static final long serialVersionUID = -1173233022912141885L; // class variables @@ -67,7 +67,7 @@ public static String hosthash(final String host, final int port) throws Malforme else if (port > 999) h = "http://" + h + ":" + port; else h = "http://" + h; } - DigestURL url = new DigestURL(h); + final DigestURL url = new DigestURL(h); return (url == null) ? null : url.hosthash(); } @@ -76,11 +76,11 @@ public static String hosthash(final String host, final int port) throws Malforme * the list is separated by comma * @param hostlist * @return list of host hashes without separation - * @throws MalformedURLException + * @throws MalformedURLException */ public static String hosthashes(final String hostlist) throws MalformedURLException { - String[] hs = CommonPattern.COMMA.split(hostlist); - StringBuilder sb = new StringBuilder(hostlist.length()); + final String[] hs = CommonPattern.COMMA.split(hostlist); + final StringBuilder sb = new StringBuilder(hostlist.length()); for (String h: hs) { if (h == null) continue; h = h.trim(); @@ -92,9 +92,9 @@ public static String hosthashes(final String hostlist) throws MalformedURLExcept return sb.toString(); } - public static Set hosthashess(String hosthashes) { + public static Set hosthashess(final String hosthashes) { if (hosthashes == null || hosthashes.isEmpty()) return null; - HashSet h = new HashSet(); + final HashSet h = new HashSet<>(); assert hosthashes.length() % 6 == 0; for (int i = 0; i < hosthashes.length(); i = i + 6) { h.add(hosthashes.substring(i, i + 6)); @@ -135,7 +135,7 @@ public DigestURL(final String url, final byte[] hash) throws MalformedURLExcepti super(url); this.hash = hash; } - + /** * DigestURI from general URI, hash already calculated * @param baseURL @@ -172,9 +172,9 @@ public static DigestURL newURL(final DigestURL baseURL, String relPath) throws M } return new DigestURL(baseURL, relPath); } - + private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful - + @Override public int hashCode() { if (this.hashCache == Integer.MIN_VALUE) { @@ -211,13 +211,13 @@ public final byte[] hash() { /** *

Extract a fragment of the url hash that can be used as a hash for the host name part of this url.

- *

WARNING : two URLs with the same host name but different protocols or ports will produce two different host hashes with this method!

+ *

WARNING : two URLs with the same host name but different protocols or ports will produce two different host hashes with this method!

* @return a 6-byte hash fragment */ public String hosthash() { return ASCII.String(this.hash(), 6, 6); } - + /** * calculated YaCy-Hash of this URI * @@ -306,7 +306,7 @@ private static final String hosthash5(final String protocol, final String host, if (host == null) { return Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(protocol)).substring(0, 5); } - boolean isIPv6HostIP = host.indexOf(':') >= 0; + final boolean isIPv6HostIP = host.indexOf(':') >= 0; final StringBuilder sb = new StringBuilder(host.length() + 15); sb.append(protocol).append(':'); if (isIPv6HostIP) {sb.append('[').append(host).append(']');} else sb.append(host); @@ -346,7 +346,7 @@ private static final String hosthash6(final String protocol, final String host, public static final String hosthash6(final String host) { return hosthash6("http", host, 80); } - + //private static String[] testTLDs = new String[] { "com", "net", "org", "uk", "fr", "de", "es", "it" }; public static final int domLengthEstimation(final byte[] urlHashBytes) { diff --git a/source/net/yacy/cora/document/id/MultiProtocolURL.java b/source/net/yacy/cora/document/id/MultiProtocolURL.java index 768ca0aa65..27e9bc2054 100644 --- a/source/net/yacy/cora/document/id/MultiProtocolURL.java +++ b/source/net/yacy/cora/document/id/MultiProtocolURL.java @@ -1,7 +1,7 @@ /** * MultiProtocolURI * Copyright 2010 by Michael Peter Christen - * First released 25.5.2010 at http://yacy.net + * First released 25.5.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ @@ -84,20 +84,20 @@ public class MultiProtocolURL implements Serializable, ComparableRegular * expressions string literals documentation */ private static final BitSet PATTERN_METACHARACTERS = new BitSet(128); - + static { // unreserved characters (chars not to escape in url) for (int i = 'A'; i <= 'Z'; i++) { // hialpha RFC1738 Section 5 @@ -134,7 +134,7 @@ public class MultiProtocolURL implements Serializable, Comparable sessionIDnames = new ConcurrentHashMap(); + private static final ConcurrentHashMap sessionIDnames = new ConcurrentHashMap<>(); public static final void initSessionIDNames(final Set idNames) { for (String s: idNames) { @@ -224,7 +224,7 @@ public MultiProtocolURL(String url) throws MalformedURLException { // identify protocol url = url.trim(); - + if (url.startsWith("//")) { // patch for urls starting with "//" which can be found in the wild url = "http:" + url; @@ -266,7 +266,7 @@ public MultiProtocolURL(String url) throws MalformedURLException { } if (q < 0) { // check for www.test.com#fragment q = url.indexOf("#", p + 3); - } + } int r; if (q < 0) { if ((r = url.indexOf('@', p + 3)) < 0) { @@ -331,14 +331,14 @@ public MultiProtocolURL(String url) throws MalformedURLException { // wrong windows path, after the doublepoint there should be a backslash. Let's add a slash, as it will be slash in the normal form h = h.substring(0, 4) + '/' + h.substring(4); } - int q = h.indexOf('/', 2); + final int q = h.indexOf('/', 2); if (q < 0 || h.length() > 3 && h.charAt(3) == ':') { // Missing root slash such as "path" or "c:/path" accepted, but the path attribute must by after all start with it - this.path = "/" + h.substring(2); + this.path = "/" + h.substring(2); } else { this.host = h.substring(2, q ); // TODO: handle "c:" ? if (this.host.equalsIgnoreCase(Domains.LOCALHOST)) this.host = null; - this.path = h.substring(q ); // "/path" + this.path = h.substring(q ); // "/path" } } else if (h.startsWith("/")) { // "/host/path" or "/host/c:/path" this.path = h; @@ -442,7 +442,7 @@ public static MultiProtocolURL newURL(final MultiProtocolURL baseURL, String rel } return new MultiProtocolURL(baseURL, relPath); } - + public MultiProtocolURL(final MultiProtocolURL baseURL, String relPath) throws MalformedURLException { if (baseURL == null) throw new MalformedURLException("base URL is null"); if (relPath == null) throw new MalformedURLException("relPath is null"); @@ -470,12 +470,12 @@ public MultiProtocolURL(final MultiProtocolURL baseURL, String relPath) throws M } else if (relPath.length() > 0 && relPath.charAt(0) == '/') { this.path = relPath; } else if (baseURL.path.endsWith("/")) { - /* According to RFC 3986 example in Appendix B. (https://tools.ietf.org/html/rfc3986) + /* According to RFC 3986 example in Appendix B. (https://tools.ietf.org/html/rfc3986) such an URL is valid : http://www.ics.uci.edu/pub/ietf/uri/#Related - - We also find similar usages in the 2016 URL living standard (https://url.spec.whatwg.org/), - for example : https://url.spec.whatwg.org/#syntax-url-absolute-with-fragment - + + We also find similar usages in the 2016 URL living standard (https://url.spec.whatwg.org/), + for example : https://url.spec.whatwg.org/#syntax-url-absolute-with-fragment + java.lang.URL constructor also accepts this form.*/ if (relPath.startsWith("/")) this.path = baseURL.path + relPath.substring(1); else this.path = baseURL.path + relPath; } else { @@ -518,7 +518,7 @@ public MultiProtocolURL(final String protocol, String host, final int port, fina identSearchpart(); escape(); } - + /** * @param host the new host to apply to the copy * @return an exact copy of this URL instance but with a new host. The original instance remains unchanged. @@ -528,22 +528,22 @@ public MultiProtocolURL ofNewHost(final String host) throws IllegalArgumentExcep if(host == null || host.trim().isEmpty()) { throw new IllegalArgumentException("Host parameter must not be null"); } - MultiProtocolURL copy = new MultiProtocolURL(this); - + final MultiProtocolURL copy = new MultiProtocolURL(this); + if (host.indexOf(':') >= 0 && host.charAt(0) != '[') { copy.host = '[' + host + ']'; // IPv6 host must be enclosed in square brackets } else { - copy.host = host; + copy.host = host; } - + if (!Punycode.isBasic(this.host)) try { this.host = toPunycode(this.host); } catch (final PunycodeException e) { ConcurrentLog.logException(e); } - + return copy; - + } /** @@ -568,7 +568,7 @@ private static final String resolveBackpath(final String path) { p = matcher.replaceAll(""); matcher.reset(p); } - /* Let's remove any eventual remaining but inappropriate '..' segments at the beginning. + /* Let's remove any eventual remaining but inappropriate '..' segments at the beginning. * See https://tools.ietf.org/html/rfc3986#section-5.2.4 -> parts 2.C and 2.D */ while(p.startsWith("/../")) { p = p.substring(3); @@ -600,7 +600,7 @@ private void escape() { * (see RFC3986, and formerly RFC1738 & RFC2396). Uses UTF-8 character codes for * non-ASCII.

*

Important : already percent-encoded characters are not re-encoded

- * + * * @param pathToEscape the path part to escape. * @return an escaped path with only ASCII characters, or null when pathToEscape * is null. @@ -612,13 +612,13 @@ private void escape() { public static String escapePath(final String pathToEscape) { return escapePath(pathToEscape, false); } - + /** *

Percent-encode/escape an URL path regular expression according to the allowed * characters in an URL path (see RFC3986) and in the {@link Pattern} regular * expressions. Uses UTF-8 character codes for non-ASCII.

*

Important : already percent-encoded characters are not re-encoded

- * + * * @param pathPattern the URL path regular expression to escape. * @return an escaped path regular expression with only allowed ASCII * characters, or null when pathPattern is null. @@ -630,7 +630,7 @@ public static String escapePath(final String pathToEscape) { public static String escapePathPattern(final String pathPattern) { return escapePath(pathPattern, true); } - + /** *

* Percent-encode/escape an URL path part according to the allowed characters @@ -642,7 +642,7 @@ public static String escapePathPattern(final String pathPattern) { * therefore meta-characters used by the {@link Pattern} class are not * percent-encoded. *

- * + * * @param pathToEscape the path part to escape. * @param isPattern when true, regular meta-characters are not escaped * @return an escaped path regular expression with only allowed ASCII @@ -661,7 +661,7 @@ private static String escapePath(final String pathToEscape, final boolean isPatt final int len = pathToEscape.length(); int i = 0; while (i < len) { - int ch = pathToEscape.charAt(i); + final int ch = pathToEscape.charAt(i); if (ch == '%' && (i + 2) < len) { final char digit1 = pathToEscape.charAt(i + 1); final char digit2 = pathToEscape.charAt(i + 2); @@ -707,7 +707,7 @@ private static String escapePath(final String pathToEscape, final boolean isPatt } return pathToEscape; } - + /** * Decode UTF-8 percent-encoded characters eventually found in the given path. *
    @@ -715,7 +715,7 @@ private static String escapePath(final String pathToEscape, final boolean isPatt *
  • the '+' character is not decoded to space character
    • *
  • no exception is thrown when invalid hexadecimal digits are found after a '%' character
    • *
- * + * * @param path an URL path eventually escaped * @return return the unescaped path or null when path is null. */ @@ -738,7 +738,7 @@ public static final String unescapePath(final String escaped) { utf8Bytes = ByteBuffer.allocate((len - i) / 3); } /* Percent-encoded character UTF-8 byte */ - int hexaValue = Integer.parseInt(escaped.substring(i + 1, i + 3), 16); + final int hexaValue = Integer.parseInt(escaped.substring(i + 1, i + 3), 16); utf8Bytes.put((byte) hexaValue); modified = true; i += 2; @@ -761,7 +761,7 @@ public static final String unescapePath(final String escaped) { return modified ? unescaped.toString() : escaped; } - + /** * @param character a character to test * @return true when the character is a valid hexadecimal digit @@ -770,7 +770,7 @@ private static boolean isHexDigit(final int character) { return (character >= '0' && character <= '9') || (character >= 'a' && character <= 'f') || (character >= 'A' && character <= 'F'); } - + private void escapeSearchpart() { final StringBuilder qtmp = new StringBuilder(this.searchpart.length() + 10); for (final Map.Entry element: getAttributes().entrySet()) { @@ -827,7 +827,7 @@ private void escapeSearchpart() { * *
  • The unreserved characters & : - _ . ! ~ * ' ( ) ; , = remain the same. * see RFC 1738 2.2 and RFC 3986 2.2 - * + * *
  • All other ASCII characters are converted into the * 3-character string "%xy", where xy is * the two-digit hexadecimal representation of the character @@ -849,24 +849,24 @@ public static StringBuilder escape(final String s) { final int ch = s.charAt(i); if (ch == ' ') { // space sbuf.append("%20"); - } else if (ch == '%') { + } else if (ch == '%') { if (i < len - 2 && s.charAt(i + 1) >= '0' && s.charAt(i + 1) <= '9' && s.charAt(i + 2) >= '0' && s.charAt(i + 2) <= '9') { // TODO: actually 0..9 A..F a..f is allowed (or any of hex[] sequence) sbuf.append((char)ch); // lets consider this is used for encoding, leave it that way } else { sbuf.append("%25"); // '%' RFC 1738 2.2 unsafe char shall be encoded } - } else if (ch == '&') { + } else if (ch == '&') { if (i < len - 6 && "amp;".equals(s.substring(i + 1, i + 5).toLowerCase(Locale.ROOT))) { sbuf.append((char)ch); // leave it that way, it is used the right way } else { sbuf.append("%26"); // this must be urlencoded } - } else if (ch == '#') { // RFC 1738 2.2 unsafe char is _not_ encoded because it may already be used for encoding + } else if (ch == '#') { // RFC 1738 2.2 unsafe char is _not_ encoded because it may already be used for encoding sbuf.append((char)ch); } else if (ch == '!' || ch == ':' // unreserved || ch == '-' || ch == '_' - || ch == '.' || ch == '~' + || ch == '.' || ch == '~' || ch == '*' || ch == '\'' || ch == '(' || ch == ')' || ch == '{' || ch == '}' @@ -903,16 +903,16 @@ public static StringBuilder escape(final String s) { public static String unescape(final String s) { try { return URLDecoder.decode(s, StandardCharsets.UTF_8.name()); - } catch (UnsupportedEncodingException e) { + } catch (final UnsupportedEncodingException e) { /* This should not happen */ ConcurrentLog.logException(e); return s; - } catch(Exception e) { + } catch(final Exception e) { /* * URLDecode may throw an IllegalArgumentException (or any other * Exception in future implementations) when the string doesn't * match the application/x-www-form-urlencoded format: in that case - * return the original string. + * return the original string. * Example case : when the valid '%' character is used in a URL but without percent encoding purpose. */ return s; @@ -1088,10 +1088,10 @@ public String getPath() { */ public String[] getPaths() { String s = (this.path == null || this.path.length() < 1) ? "" : this.path.charAt(0) == '/' ? this.path.substring(1) : this.path; - int p = s.lastIndexOf('/'); + final int p = s.lastIndexOf('/'); if (p < 0) return new String[0]; s = s.substring(0, p); // the paths do not contain the last part, which is considered as the getFileName() part. - String[] paths = CommonPattern.SLASH.split(s); + final String[] paths = CommonPattern.SLASH.split(s); return paths; } @@ -1106,7 +1106,7 @@ public File getLocalFile() { char c = this.path.charAt(1); if (c == ':') return new File(this.path); if (c == '|') return new File(this.path.charAt(0) + ":" + this.path.substring(2)); - + if (this.path.length() > 1) { // prevent StringIndexOutOfBoundsException c = this.path.charAt(2); if (c == ':' || c == '|') return new File(this.path.charAt(1) + ":" + this.path.substring(3)); @@ -1124,12 +1124,12 @@ public String getAuthority() { public String getHost() { return this.host; } - + public String getOrganization() { - String dnc = Domains.getDNC(host); - String subdomOrga = host.length() - dnc.length() <= 0 ? "" : host.substring(0, host.length() - dnc.length() - 1); - int p = subdomOrga.lastIndexOf('.'); - String orga = (p < 0) ? subdomOrga : subdomOrga.substring(p + 1); + final String dnc = Domains.getDNC(this.host); + final String subdomOrga = this.host.length() - dnc.length() <= 0 ? "" : this.host.substring(0, this.host.length() - dnc.length() - 1); + final int p = subdomOrga.lastIndexOf('.'); + final String orga = (p < 0) ? subdomOrga : subdomOrga.substring(p + 1); return orga; } @@ -1138,7 +1138,7 @@ public String getOrganization() { */ public String getTLD() { if (this.host == null) return ""; - int p = this.host.lastIndexOf('.'); + final int p = this.host.lastIndexOf('.'); if (p < 0) return ""; return this.host.substring(p + 1); } @@ -1161,7 +1161,7 @@ public String getProtocol() { /** * @return this URL fragment or null if has no fragment * @see URL fragment concept at WHATWG - * @see URL fragment section in RFC 3986 + * @see URL fragment section in RFC 3986 */ public String getRef() { return this.anchor; @@ -1194,10 +1194,10 @@ public String getSearchpart() { public Map getSearchpartMap() { if (this.searchpart == null) return null; this.searchpart = this.searchpart.replaceAll("&", "&"); - String[] parts = CommonPattern.AMP.split(this.searchpart); - Map map = new LinkedHashMap(); - for (String part: parts) { - int p = part.indexOf('='); + final String[] parts = CommonPattern.AMP.split(this.searchpart); + final Map map = new LinkedHashMap<>(); + for (final String part: parts) { + final int p = part.indexOf('='); if (p > 0) map.put(part.substring(0, p), part.substring(p + 1)); else map.put(part, ""); } return map; @@ -1210,7 +1210,7 @@ public String toString() { /** * Tokenizes url as string (without the protocol). - * For example "http://host.com/path/file.txt" returns "host com path file ext" + * For example "http://host.com/path/file.txt" returns "host com path file ext" * @return url tokens as one string */ public String toTokens() { @@ -1233,7 +1233,7 @@ public static String toTokens(final String s) { // split the string into tokens and add all camel-case splitting final String[] u = CommonPattern.SPACES.split(sb); - final Set token = new LinkedHashSet(); + final Set token = new LinkedHashSet<>(); for (final String r: u) token.add(r); for (final String r: u) token.addAll(parseCamelCase(r)); @@ -1246,7 +1246,7 @@ public static String toTokens(final String s) { public static enum CharType { low, high, number; } private static Set parseCamelCase(String s) { - final Set token = new LinkedHashSet(); + final Set token = new LinkedHashSet<>(); if (s.isEmpty()) return token; int p = 0; CharType type = charType(s.charAt(0)), nct = type; @@ -1272,18 +1272,18 @@ private static Set parseCamelCase(String s) { /** * Evaluates url search part and returns attribute '=' value pairs * the returned values are in clear text (without urlencoding). - * + * * To get the parameter map as (url-encoded key and values) * @see getSearchpartMap() * * @return map key=attribue name, value=string after '=' */ public Map getAttributes() { - Map map = new LinkedHashMap<>(); + final Map map = new LinkedHashMap<>(); if (this.searchpart == null) return map; final String[] questp = CommonPattern.AMP.split(this.searchpart, -1); for (final String element : questp) { - int p = element.indexOf('='); + final int p = element.indexOf('='); if (p != -1) { map.put(unescape(element.substring(0, p)), unescape(element.substring(p + 1))); } else { @@ -1292,13 +1292,13 @@ public Map getAttributes() { } return map; } - + private static CharType charType(final char c) { if (Character.isLowerCase(c)) return CharType.low; if (Character.isDigit(c)) return CharType.number; return CharType.high; } - + public String toNormalform(final boolean excludeAnchor) { return toNormalform(excludeAnchor, false); } @@ -1326,7 +1326,7 @@ public String toNormalform(final boolean excludeAnchor, final boolean removeSess defaultPort = true; } String urlPath = this.getFile(excludeAnchor, removeSessionID); - String h = getHost(); + final String h = getHost(); final StringBuilder u = new StringBuilder(20 + (urlPath == null ? 0 : urlPath.length()) + ((h == null) ? 0 : h.length())); u.append(this.protocol); u.append("://"); @@ -1345,8 +1345,8 @@ public String toNormalform(final boolean excludeAnchor, final boolean removeSess urlPath = urlPath.replace('\\', '/'); } u.append(urlPath); - String result = u.toString(); - + final String result = u.toString(); + return result; } @@ -1375,8 +1375,8 @@ public String urlstub(final boolean excludeAnchor, final boolean removeSessionID } else if (isFile()) { defaultPort = true; } - String urlPath = this.getFile(excludeAnchor, removeSessionID); - String h = getHost(); + final String urlPath = this.getFile(excludeAnchor, removeSessionID); + final String h = getHost(); final StringBuilder u = new StringBuilder(20 + urlPath.length() + ((h == null) ? 0 : h.length())); if (h != null) { if (this.userInfo != null && !(this.isFTP() && this.userInfo.startsWith(FTPClient.ANONYMOUS))) { @@ -1390,8 +1390,8 @@ public String urlstub(final boolean excludeAnchor, final boolean removeSessionID u.append(this.port); } u.append(urlPath); - String result = u.toString(); - + final String result = u.toString(); + return result; } @@ -1491,12 +1491,12 @@ public final String language() { String language = "en"; if (this.host == null) return language; final int pos = this.host.lastIndexOf('.'); - String host_tld = this.host.substring(pos + 1).toLowerCase(Locale.ROOT); + final String host_tld = this.host.substring(pos + 1).toLowerCase(Locale.ROOT); if (pos == 0) return language; - int length = this.host.length() - pos - 1; + final int length = this.host.length() - pos - 1; switch (length) { case 2: - char firstletter = host_tld.charAt(0); + final char firstletter = host_tld.charAt(0); switch (firstletter) {//speed-up case 'a': if (host_tld.equals("au")) {//Australia /91,000,000 @@ -2500,7 +2500,7 @@ public String getName() throws IOException { /** * Get directory listing of file or smb url * respects the hidden attribute of a directory (return null if hidden) - * + * * @return names of files and directories or null * @throws IOException */ @@ -2554,11 +2554,11 @@ public InputStream getInputStream(final ClientIdentification.Agent agent) throws public byte[] get(final ClientIdentification.Agent agent, final String username, final String pass) throws IOException { if (isFile()) { - byte[] b = read(new FileInputStream(getFSFile())); + final byte[] b = read(new FileInputStream(getFSFile())); return b; } if (isSMB()) { - byte[] b = read(new SmbFileInputStream(getSmbFile())); + final byte[] b = read(new SmbFileInputStream(getSmbFile())); return b; } if (isFTP()) { @@ -2589,19 +2589,19 @@ public boolean exists(final ClientIdentification.Agent agent) { if (isFTP()) { final FTPClient client = new FTPClient(); client.open(this.host, this.port < 0 ? 21 : this.port); - return client.fileSize(path) > 0; + return client.fileSize(this.path) > 0; } if (isHTTP() || isHTTPS()) { final HTTPClient client = new HTTPClient(agent); client.setHost(getHost()); - org.apache.http.HttpResponse response = client.HEADResponse(this, true); + final org.apache.http.HttpResponse response = client.HEADResponse(this, true); client.close(); if (response == null) return false; - int status = response.getStatusLine().getStatusCode(); + final int status = response.getStatusLine().getStatusCode(); return status == 200 || status == 301 || status == 302; } return false; - } catch (IOException e) { + } catch (final IOException e) { if (e.getMessage().contains("Circular redirect to")) return true; // exception; this is a 302 which the client actually accepts //e.printStackTrace(); return false; @@ -2626,7 +2626,7 @@ public static byte[] read(final InputStream source) throws IOException { } finally { try { source.close(); - } catch(IOException ignored) { + } catch(final IOException ignored) { } } } @@ -2654,7 +2654,7 @@ public static String[] urlComps(String normalizedURL) { // TODO lowering case in a locale sensitive manner makes sense here, but the used language locale should not dependant on the default system locale return splitpattern.split(normalizedURL.toLowerCase()); // word components of the url } - + public static void main(final String[] args) { final String[][] test = new String[][]{ new String[]{null, "file://y:/"}, @@ -2717,7 +2717,7 @@ public static void main(final String[] args) { String environment, url; MultiProtocolURL aURL, aURL1; java.net.URL jURL; - for (String[] element : test) { + for (final String[] element : test) { environment = element[0]; url = element[1]; try {aURL = MultiProtocolURL.newURL(environment, url);} catch (final MalformedURLException e) {e.printStackTrace(); aURL = null;} diff --git a/source/net/yacy/cora/federate/AbstractFederateSearchConnector.java b/source/net/yacy/cora/federate/AbstractFederateSearchConnector.java index 71bce09509..f976c16605 100644 --- a/source/net/yacy/cora/federate/AbstractFederateSearchConnector.java +++ b/source/net/yacy/cora/federate/AbstractFederateSearchConnector.java @@ -1,7 +1,7 @@ /** * AbstractFederateSearchConnector.java * Copyright 2015 by Burkhard Buelte - * First released 19.01.2015 at http://yacy.net + * First released 19.01.2015 at https://yacy.net * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free diff --git a/source/net/yacy/cora/federate/FederateSearchConnector.java b/source/net/yacy/cora/federate/FederateSearchConnector.java index 4f7ebfa8e4..7b18b4ce4d 100644 --- a/source/net/yacy/cora/federate/FederateSearchConnector.java +++ b/source/net/yacy/cora/federate/FederateSearchConnector.java @@ -1,7 +1,7 @@ /** * FederateSearchConnector.java * Copyright 2015 by Burkhard Buelte - * First released 19.01.2015 at http://yacy.net + * First released 19.01.2015 at https://yacy.net * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free diff --git a/source/net/yacy/cora/federate/FederateSearchManager.java b/source/net/yacy/cora/federate/FederateSearchManager.java index 7a0557d049..e4fdb6458c 100644 --- a/source/net/yacy/cora/federate/FederateSearchManager.java +++ b/source/net/yacy/cora/federate/FederateSearchManager.java @@ -1,7 +1,7 @@ /** * FederateSearchManager.java * Copyright 2015 by Burkhard Buelte - * First released 19.01.2015 at http://yacy.net + * First released 19.01.2015 at https://yacy.net * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free diff --git a/source/net/yacy/cora/federate/SolrFederateSearchConnector.java b/source/net/yacy/cora/federate/SolrFederateSearchConnector.java index 811bfaf080..ef51a2360d 100644 --- a/source/net/yacy/cora/federate/SolrFederateSearchConnector.java +++ b/source/net/yacy/cora/federate/SolrFederateSearchConnector.java @@ -1,7 +1,7 @@ /** * SolrFederateSearchConnector.java * Copyright 2015 by Burkhard Buelte - * First released 19.01.2015 at http://yacy.net + * First released 19.01.2015 at https://yacy.net * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free diff --git a/source/net/yacy/cora/federate/opensearch/OpenSearchConnector.java b/source/net/yacy/cora/federate/opensearch/OpenSearchConnector.java index d61a9deed3..96b8c821b7 100644 --- a/source/net/yacy/cora/federate/opensearch/OpenSearchConnector.java +++ b/source/net/yacy/cora/federate/opensearch/OpenSearchConnector.java @@ -1,7 +1,7 @@ /** * OpenSearchConnector * Copyright 2012 by Michael Peter Christen - * First released 03.11.2012 at http://yacy.net + * First released 03.11.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -60,13 +60,13 @@ * configured systems until number of needed results are available. */ public class OpenSearchConnector extends AbstractFederateSearchConnector implements FederateSearchConnector { - - /** + + /** * HTML mapping properties used to retrieve result from HTML when the results * are not provided as a standard RSS/Atom feed but as simple HTML. */ - private Properties htmlMapping; - + private final Properties htmlMapping; + /** * @param instanceName open search instance name * @return the html mapping configuration file name derived from the instance name @@ -74,16 +74,16 @@ public class OpenSearchConnector extends AbstractFederateSearchConnector impleme public static String htmlMappingFileName(final String instanceName) { return instanceName + ".html.map.properties"; } - + /** - * @param urlTemplate OpenSearch URL template + * @param urlTemplate OpenSearch URL template */ public OpenSearchConnector(final String urlTemplate) { super(); this.baseurl = urlTemplate; this.htmlMapping = new Properties(); } - + @Override public boolean init(final String name, final String cfgFileName) { this.instancename = name; @@ -94,13 +94,13 @@ public boolean init(final String name, final String cfgFileName) { try { cfgFileStream = new BufferedInputStream(new FileInputStream(cfgFileName)); this.htmlMapping.load(cfgFileStream); - } catch (IOException e) { + } catch (final IOException e) { ConcurrentLog.config("OpenSearchConnector." + this.instancename, "Error reading html mapping file : " + cfgFileName, e); } finally { if (cfgFileStream != null) { try { cfgFileStream.close(); - } catch (IOException e) { + } catch (final IOException e) { ConcurrentLog.config("OpenSearchConnector." + this.instancename, "Error closing html mapping file : " + cfgFileName, e); } } @@ -108,11 +108,11 @@ public boolean init(final String name, final String cfgFileName) { } return true; } - + /** * replace Opensearchdescription search template parameter with actual values */ - private String parseSearchTemplate(String searchurltemplate, String query, int start, int rows) { + private String parseSearchTemplate(final String searchurltemplate, final String query, final int start, final int rows) { String tmps = searchurltemplate.replaceAll("\\?}", "}"); // some optional parameters may include question mark '{param?}=' tmps = tmps.replace("{startIndex}", Integer.toString(start)); tmps = tmps.replace("{startPage}", ""); @@ -127,42 +127,42 @@ private String parseSearchTemplate(String searchurltemplate, String query, int s * @param linkElement html link result node. Must not be null. * @return and {@link URIMetadataNode} instance from the html link element or null when minimum required information is missing or malformed */ - protected URIMetadataNode htmlLinkToMetadataNode(Element linkElement) { + protected URIMetadataNode htmlLinkToMetadataNode(final Element linkElement) { URIMetadataNode doc = null; - String absoluteURL = linkElement.absUrl("href"); + final String absoluteURL = linkElement.absUrl("href"); try { if (!absoluteURL.isEmpty()) { - DigestURL uri = new DigestURL(absoluteURL); + final DigestURL uri = new DigestURL(absoluteURL); doc = new URIMetadataNode(uri); - + if(linkElement.hasText() && !this.htmlMapping.containsKey("title")) { /* Let's use the link text as default title when no mapping is defined.*/ doc.setField(CollectionSchema.title.getSolrFieldName(), linkElement.text()); } - - String targetLang = linkElement.attr("hreflang"); + + final String targetLang = linkElement.attr("hreflang"); if(targetLang != null && !targetLang.isEmpty()) { doc.setField(CollectionSchema.language_s.getSolrFieldName(), targetLang); } - + final String mime = TextParser.mimeOf(uri); if (mime != null) { doc.setField(CollectionSchema.content_type.getSolrFieldName(), mime); } - + /* * add collection "dht" which is used to differentiate metadata * from full crawl data in the index */ doc.setField(CollectionSchema.collection_sxt.getSolrFieldName(), "dht"); } - } catch (MalformedURLException e) { + } catch (final MalformedURLException e) { ConcurrentLog.fine("OpenSearchConnector." + this.instancename, "Malformed url : " + absoluteURL); } return doc; } - + /** * Extract results from the HTML result stream, using the html mapping properties. * Important : it is the responsibility of the caller to close the stream. @@ -171,33 +171,33 @@ protected URIMetadataNode htmlLinkToMetadataNode(Element linkElement) { * @return a list of URI nodes, eventually empty. * @throws IOException when a read/write exception occurred */ - protected List parseHTMLResult(InputStream resultStream, String charsetName) throws IOException { - List docs = new ArrayList<>(); - String resultSelector = this.htmlMapping.getProperty("_result"); - String skuSelector = this.htmlMapping.getProperty("_sku"); + protected List parseHTMLResult(final InputStream resultStream, final String charsetName) throws IOException { + final List docs = new ArrayList<>(); + final String resultSelector = this.htmlMapping.getProperty("_result"); + final String skuSelector = this.htmlMapping.getProperty("_sku"); if (resultSelector == null || skuSelector == null) { ConcurrentLog.warn("OpenSearchConnector." + this.instancename, "HTML mapping is incomplete!"); return docs; } - Document jsoupDoc = Jsoup.parse(resultStream, charsetName, this.baseurl); - Elements results = jsoupDoc.select(resultSelector); + final Document jsoupDoc = Jsoup.parse(resultStream, charsetName, this.baseurl); + final Elements results = jsoupDoc.select(resultSelector); - for (Element result : results) { - Elements skuNodes = result.select(skuSelector); + for (final Element result : results) { + final Elements skuNodes = result.select(skuSelector); if (!skuNodes.isEmpty()) { Element skuNode = skuNodes.first(); if (!"a".equals(skuNode.tagName())) { /* * The selector may refer to a node with link(s) inside */ - Elements links = skuNode.select("a[href]"); + final Elements links = skuNode.select("a[href]"); if (!links.isEmpty()) { skuNode = links.first(); } } if (skuNode.hasAttr("href")) { - URIMetadataNode newDoc = htmlLinkToMetadataNode(skuNode); + final URIMetadataNode newDoc = htmlLinkToMetadataNode(skuNode); if (newDoc != null) { /* Let's handle other field mappings */ htmlResultToFields(result, newDoc); @@ -214,32 +214,32 @@ protected List parseHTMLResult(InputStream resultStream, String * @param resultNode html single result node * @param newdoc result document to fill */ - private void htmlResultToFields(Element resultNode, URIMetadataNode newdoc) { - for (Entry entry : this.htmlMapping.entrySet()) { + private void htmlResultToFields(final Element resultNode, final URIMetadataNode newdoc) { + for (final Entry entry : this.htmlMapping.entrySet()) { if (entry.getKey() instanceof String && entry.getValue() instanceof String) { - String yacyFieldName = (String) entry.getKey(); - String selector = (String) entry.getValue(); - + final String yacyFieldName = (String) entry.getKey(); + final String selector = (String) entry.getValue(); + if (!yacyFieldName.startsWith("_")) { /* If Switchboard environment is set, check the index configuration has this field enabled */ if (Switchboard.getSwitchboard() == null || Switchboard.getSwitchboard().index == null || Switchboard.getSwitchboard().index.fulltext().getDefaultConfiguration() .contains(yacyFieldName)) { - Elements nodes = resultNode.select(selector); + final Elements nodes = resultNode.select(selector); SchemaDeclaration est; try { est = CollectionSchema.valueOf(yacyFieldName); - } catch(IllegalArgumentException e) { + } catch(final IllegalArgumentException e) { ConcurrentLog.config("OpenSearchConnector." + this.instancename, "Ignored " + yacyFieldName + " field mapping : not a field of this schema."); continue; } if (est.isMultiValued()) { if (!nodes.isEmpty()) { - for (Element node : nodes) { - String value = node.text(); + for (final Element node : nodes) { + final String value = node.text(); if (!value.isEmpty()) { newdoc.addField(yacyFieldName, value); } @@ -247,8 +247,8 @@ private void htmlResultToFields(Element resultNode, URIMetadataNode newdoc) { } } else { if (!nodes.isEmpty()) { - Element node = nodes.first(); - String value = node.text(); + final Element node = nodes.first(); + final String value = node.text(); if (!value.isEmpty()) { /* Perform eventual type conversion */ try { @@ -257,7 +257,7 @@ private void htmlResultToFields(Element resultNode, URIMetadataNode newdoc) { } else { newdoc.setField(yacyFieldName, value); } - } catch (NumberFormatException ex) { + } catch (final NumberFormatException ex) { continue; } } @@ -278,11 +278,11 @@ private void htmlResultToFields(Element resultNode, URIMetadataNode newdoc) { * @return query results (metadata) with fields according to YaCy schema */ @Override - public List query(QueryParams query) { + public List query(final QueryParams query) { return query(query.getQueryGoal().getQueryString(false), 0, query.itemsPerPage); } - + /** * Query the remote system at baseurl with the specified search terms * @param searchTerms search terms @@ -291,17 +291,17 @@ public List query(QueryParams query) { * @return a result list eventually empty when no results where found or when an error occured */ public List query(final String searchTerms, final int startIndex, final int count) { - List docs = new ArrayList(); - + List docs = new ArrayList<>(); + // see http://www.loc.gov/standards/sru/ - String searchurl = this.parseSearchTemplate(baseurl, searchTerms, startIndex, count); + final String searchurl = this.parseSearchTemplate(this.baseurl, searchTerms, startIndex, count); try { - DigestURL aurl = new DigestURL(searchurl); + final DigestURL aurl = new DigestURL(searchurl); try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) { this.lastaccesstime = System.currentTimeMillis(); - - byte[] result = httpClient.GETbytes(aurl, null, null, false); - + + final byte[] result = httpClient.GETbytes(aurl, null, null, false); + if(result == null) { String details; if(httpClient.getHttpResponse() != null && httpClient.getHttpResponse().getStatusLine() != null) { @@ -324,15 +324,15 @@ public List query(final String searchTerms, final int startInde } } else { /* Other mime types or unknown : let's try to parse the result as RSS or Atom Feed */ - RSSReader rssReader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result); + final RSSReader rssReader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result); if (rssReader != null) { final RSSFeed feed = rssReader.getFeed(); if (feed != null) { for (final RSSMessage item : feed) { try { - DigestURL uri = new DigestURL(item.getLink()); + final DigestURL uri = new DigestURL(item.getLink()); - URIMetadataNode doc = new URIMetadataNode(uri); + final URIMetadataNode doc = new URIMetadataNode(uri); doc.setField(CollectionSchema.charset_s.getSolrFieldName(), StandardCharsets.UTF_8.name()); doc.setField(CollectionSchema.author.getSolrFieldName(), item.getAuthor()); doc.setField(CollectionSchema.title.getSolrFieldName(), item.getTitle()); @@ -369,16 +369,16 @@ public List query(final String searchTerms, final int startInde } } } - } catch (IOException ex) { + } catch (final IOException ex) { ConcurrentLog.logException(ex); ConcurrentLog.info("OpenSearchConnector." + this.instancename, "no connection to " + searchurl); } - } catch (MalformedURLException ee) { + } catch (final MalformedURLException ee) { ConcurrentLog.warn("OpenSearchConnector." + this.instancename, "malformed url " + searchurl); } return docs; } - + /** * Main procedure : can be used to test results retrieval from an open search system * @param args main arguments list: @@ -388,14 +388,14 @@ public List query(final String searchTerms, final int startInde *
  • Html mapping file path (optional)
    • * */ - public static void main(String args[]) { + public static void main(final String args[]) { try { if (args.length < 2) { System.out.println("Usage : java " + OpenSearchConnector.class.getCanonicalName() + " <\"searchTerms\"> [htmlMappingFile]"); return; } - OpenSearchConnector connector = new OpenSearchConnector(args[0]); + final OpenSearchConnector connector = new OpenSearchConnector(args[0]); String htmlMappingFile; if (args.length > 2) { htmlMappingFile = args[2]; @@ -407,12 +407,12 @@ public static void main(String args[]) { if(searchTerms.length() > 2 && searchTerms.startsWith("\"") && searchTerms.endsWith("\"")) { searchTerms = searchTerms.substring(1, searchTerms.length() - 1); } - List docs = connector.query(searchTerms, 0, 20); + final List docs = connector.query(searchTerms, 0, 20); if (docs.isEmpty()) { System.out.println("No results"); } else { - for (URIMetadataNode doc : docs) { + for (final URIMetadataNode doc : docs) { System.out.println("title : " + doc.getFieldValue(CollectionSchema.title.getSolrFieldName())); System.out.println("sku : " + doc.getFieldValue(CollectionSchema.sku.getSolrFieldName())); System.out.println( diff --git a/source/net/yacy/cora/federate/opensearch/SRURSSConnector.java b/source/net/yacy/cora/federate/opensearch/SRURSSConnector.java index a8e1ebb150..e217f75d03 100644 --- a/source/net/yacy/cora/federate/opensearch/SRURSSConnector.java +++ b/source/net/yacy/cora/federate/opensearch/SRURSSConnector.java @@ -1,7 +1,7 @@ /** * AccumulateSRURSS * Copyright 2010 by Michael Peter Christen - * First released 06.01.2011 at http://yacy.net + * First released 06.01.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -25,6 +25,8 @@ import java.util.LinkedHashMap; import java.util.concurrent.BlockingQueue; +import org.apache.http.entity.mime.content.ContentBody; + import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.document.feed.RSSFeed; import net.yacy.cora.document.feed.RSSMessage; @@ -34,8 +36,6 @@ import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.http.HTTPClient; -import org.apache.http.entity.mime.content.ContentBody; - public class SRURSSConnector { private final static int recordsPerSession = 250; @@ -114,7 +114,7 @@ public static RSSFeed loadSRURSS( // send request byte[] result = new byte[0]; try { - final LinkedHashMap parts = new LinkedHashMap(); + final LinkedHashMap parts = new LinkedHashMap<>(); parts.put("query", UTF8.StringBody(query)); parts.put("startRecord", UTF8.StringBody(Integer.toString(startRecord))); parts.put("maximumRecords", UTF8.StringBody(Long.toString(maximumRecords))); diff --git a/source/net/yacy/cora/federate/solr/FailCategory.java b/source/net/yacy/cora/federate/solr/FailCategory.java index cad47f4614..31d27d748c 100644 --- a/source/net/yacy/cora/federate/solr/FailCategory.java +++ b/source/net/yacy/cora/federate/solr/FailCategory.java @@ -1,7 +1,7 @@ /** * FailCategory * Copyright 2013 by Michael Peter Christen - * First released 17.10.2013 at http://yacy.net + * First released 17.10.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/FailType.java b/source/net/yacy/cora/federate/solr/FailType.java index 59a57e7f8e..c3531b4971 100644 --- a/source/net/yacy/cora/federate/solr/FailType.java +++ b/source/net/yacy/cora/federate/solr/FailType.java @@ -1,7 +1,7 @@ /** * FailType * Copyright 2012 by Michael Peter Christen - * First released 23.11.2012 at http://yacy.net + * First released 23.11.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/ProcessType.java b/source/net/yacy/cora/federate/solr/ProcessType.java index 8c64da5b3d..803f703cae 100644 --- a/source/net/yacy/cora/federate/solr/ProcessType.java +++ b/source/net/yacy/cora/federate/solr/ProcessType.java @@ -1,7 +1,7 @@ /** * ProcessType * Copyright 2013 by Michael Peter Christen - * First released 02.01.2013 at http://yacy.net + * First released 02.01.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/Ranking.java b/source/net/yacy/cora/federate/solr/Ranking.java index ee82c887d9..a16e3b56e2 100644 --- a/source/net/yacy/cora/federate/solr/Ranking.java +++ b/source/net/yacy/cora/federate/solr/Ranking.java @@ -1,7 +1,7 @@ /** * Ranking * Copyright 2013 by Michael Peter Christen - * First released 12.03.2013 at http://yacy.net + * First released 12.03.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/SchemaConfiguration.java b/source/net/yacy/cora/federate/solr/SchemaConfiguration.java index 175effcc08..bf2edc6862 100644 --- a/source/net/yacy/cora/federate/solr/SchemaConfiguration.java +++ b/source/net/yacy/cora/federate/solr/SchemaConfiguration.java @@ -1,7 +1,7 @@ /** * SchemaConfiguration * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 29.06.2011 at http://yacy.net + * First released 29.06.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/SchemaDeclaration.java b/source/net/yacy/cora/federate/solr/SchemaDeclaration.java index 94767e64c5..1460a1d512 100644 --- a/source/net/yacy/cora/federate/solr/SchemaDeclaration.java +++ b/source/net/yacy/cora/federate/solr/SchemaDeclaration.java @@ -1,7 +1,7 @@ /** * SchemaDeclaration * Copyright 2011 by Michael Peter Christen - * First released 14.04.2011 at http://yacy.net + * First released 14.04.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/SolrType.java b/source/net/yacy/cora/federate/solr/SolrType.java index 1b844a9034..70694e3c78 100644 --- a/source/net/yacy/cora/federate/solr/SolrType.java +++ b/source/net/yacy/cora/federate/solr/SolrType.java @@ -1,7 +1,7 @@ /** * SolrType * Copyright 2011 by Michael Peter Christen - * First released 14.04.2011 at http://yacy.net + * First released 14.04.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java index b284e876c8..e2e9ea0ecc 100644 --- a/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java @@ -1,7 +1,7 @@ /** * AbstractSolrConnector * Copyright 2012 by Michael Peter Christen - * First released 27.06.2012 at http://yacy.net + * First released 27.06.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java index b1a931acd8..1b0728727b 100644 --- a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java @@ -1,7 +1,7 @@ /** * EmbeddedSolrConnector * Copyright 2012 by Michael Peter Christen - * First released 21.06.2012 at http://yacy.net + * First released 21.06.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java index b46d13193c..97e5b402e7 100644 --- a/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java @@ -1,7 +1,7 @@ /** * MirrorSolrConnector * Copyright 2013 by Michael Peter Christen - * First released 18.02.2012 at http://yacy.net + * First released 18.02.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java index 645c301f9f..7ab94cb2e1 100644 --- a/source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java @@ -1,7 +1,7 @@ /** * SolrSingleConnector * Copyright 2011 by Michael Peter Christen - * First released 14.04.2011 at http://yacy.net + * First released 14.04.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/connector/ShardSelection.java b/source/net/yacy/cora/federate/solr/connector/ShardSelection.java index c05afd3120..7c07124a54 100644 --- a/source/net/yacy/cora/federate/solr/connector/ShardSelection.java +++ b/source/net/yacy/cora/federate/solr/connector/ShardSelection.java @@ -1,7 +1,7 @@ /** * SolrChardingSelection * Copyright 2011 by Michael Peter Christen - * First released 25.05.2011 at http://yacy.net + * First released 25.05.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/connector/SolrConnector.java b/source/net/yacy/cora/federate/solr/connector/SolrConnector.java index a0de172305..b9706fb35f 100644 --- a/source/net/yacy/cora/federate/solr/connector/SolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/SolrConnector.java @@ -1,7 +1,7 @@ /** * SolrConnector * Copyright 2011 by Michael Peter Christen - * First released 13.09.2011 at http://yacy.net + * First released 13.09.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java b/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java index 4a1b7c4fbe..9fabc4d8eb 100644 --- a/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java @@ -1,7 +1,7 @@ /** * SolrServerConnector * Copyright 2012 by Michael Peter Christen - * First released 21.06.2012 at http://yacy.net + * First released 21.06.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/instance/EmbeddedInstance.java b/source/net/yacy/cora/federate/solr/instance/EmbeddedInstance.java index 4ffa81ceff..cd453f0579 100644 --- a/source/net/yacy/cora/federate/solr/instance/EmbeddedInstance.java +++ b/source/net/yacy/cora/federate/solr/instance/EmbeddedInstance.java @@ -1,7 +1,7 @@ /** * EmbeddedInstance * Copyright 2013 by Michael Peter Christen - * First released 13.02.2013 at http://yacy.net + * First released 13.02.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java b/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java index bb48a4d5ab..61e4321e6d 100644 --- a/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java +++ b/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java @@ -1,7 +1,7 @@ /** * InstanceMirror * Copyright 2013 by Michael Peter Christen - * First released 18.02.2013 at http://yacy.net + * First released 18.02.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java b/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java index 39a0adb4fc..0052eb2f48 100644 --- a/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java +++ b/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java @@ -1,7 +1,7 @@ /** * RemoteInstance * Copyright 2013 by Michael Peter Christen - * First released 13.02.2013 at http://yacy.net + * First released 13.02.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/instance/ResponseAccumulator.java b/source/net/yacy/cora/federate/solr/instance/ResponseAccumulator.java index 9ef9b67108..71afe3fb00 100644 --- a/source/net/yacy/cora/federate/solr/instance/ResponseAccumulator.java +++ b/source/net/yacy/cora/federate/solr/instance/ResponseAccumulator.java @@ -1,7 +1,7 @@ /** * ResponseAccumulator * Copyright 2013 by Michael Peter Christen - * First released 13.02.2013 at http://yacy.net + * First released 13.02.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/instance/ServerMirror.java b/source/net/yacy/cora/federate/solr/instance/ServerMirror.java index 523cd5a585..3ecc57b532 100644 --- a/source/net/yacy/cora/federate/solr/instance/ServerMirror.java +++ b/source/net/yacy/cora/federate/solr/instance/ServerMirror.java @@ -1,7 +1,7 @@ /** * ServerMirror * Copyright 2013 by Michael Peter Christen - * First released 18.02.2013 at http://yacy.net + * First released 18.02.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/instance/ServerShard.java b/source/net/yacy/cora/federate/solr/instance/ServerShard.java index fc93298542..65be7b6ebd 100644 --- a/source/net/yacy/cora/federate/solr/instance/ServerShard.java +++ b/source/net/yacy/cora/federate/solr/instance/ServerShard.java @@ -1,7 +1,7 @@ /** * ServerShard * Copyright 2013 by Michael Peter Christen - * First released 13.02.2013 at http://yacy.net + * First released 13.02.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/instance/ShardInstance.java b/source/net/yacy/cora/federate/solr/instance/ShardInstance.java index b955391d26..2244be4c21 100644 --- a/source/net/yacy/cora/federate/solr/instance/ShardInstance.java +++ b/source/net/yacy/cora/federate/solr/instance/ShardInstance.java @@ -1,7 +1,7 @@ /** * ShardInstance * Copyright 2013 by Michael Peter Christen - * First released 13.02.2013 at http://yacy.net + * First released 13.02.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/instance/SolrInstance.java b/source/net/yacy/cora/federate/solr/instance/SolrInstance.java index dd16c69efc..70285108b8 100644 --- a/source/net/yacy/cora/federate/solr/instance/SolrInstance.java +++ b/source/net/yacy/cora/federate/solr/instance/SolrInstance.java @@ -1,7 +1,7 @@ /** * SolrInstance * Copyright 2013 by Michael Peter Christen - * First released 13.02.2013 at http://yacy.net + * First released 13.02.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/logic/AbstractTerm.java b/source/net/yacy/cora/federate/solr/logic/AbstractTerm.java index 18bb7f3f4f..a88b1cd2a2 100644 --- a/source/net/yacy/cora/federate/solr/logic/AbstractTerm.java +++ b/source/net/yacy/cora/federate/solr/logic/AbstractTerm.java @@ -1,7 +1,7 @@ /** * AbstractTerm * Copyright 2014 by Michael Peter Christen - * First released 03.08.2014 at http://yacy.net + * First released 03.08.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/logic/BooleanLiteral.java b/source/net/yacy/cora/federate/solr/logic/BooleanLiteral.java index 47b5f8b373..46d9e7d3a1 100644 --- a/source/net/yacy/cora/federate/solr/logic/BooleanLiteral.java +++ b/source/net/yacy/cora/federate/solr/logic/BooleanLiteral.java @@ -1,7 +1,7 @@ /** * BooleanLiteral * Copyright 2014 by Michael Peter Christen - * First released 24.10.2014 at http://yacy.net + * First released 24.10.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/logic/CatchallLiteral.java b/source/net/yacy/cora/federate/solr/logic/CatchallLiteral.java index 1a9aa5e0d8..02104a42fe 100644 --- a/source/net/yacy/cora/federate/solr/logic/CatchallLiteral.java +++ b/source/net/yacy/cora/federate/solr/logic/CatchallLiteral.java @@ -1,7 +1,7 @@ /** * CatchallLiteral * Copyright 2014 by Michael Peter Christen - * First released 24.10.2014 at http://yacy.net + * First released 24.10.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/logic/Conjunction.java b/source/net/yacy/cora/federate/solr/logic/Conjunction.java index 06f09204ec..294e7471e2 100644 --- a/source/net/yacy/cora/federate/solr/logic/Conjunction.java +++ b/source/net/yacy/cora/federate/solr/logic/Conjunction.java @@ -1,7 +1,7 @@ /** * Conjunction * Copyright 2014 by Michael Peter Christen - * First released 03.08.2014 at http://yacy.net + * First released 03.08.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/logic/Disjunction.java b/source/net/yacy/cora/federate/solr/logic/Disjunction.java index bd2c74073b..ea11b02b40 100644 --- a/source/net/yacy/cora/federate/solr/logic/Disjunction.java +++ b/source/net/yacy/cora/federate/solr/logic/Disjunction.java @@ -1,7 +1,7 @@ /** * Disjunction * Copyright 2014 by Michael Peter Christen - * First released 03.08.2014 at http://yacy.net + * First released 03.08.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/logic/Literal.java b/source/net/yacy/cora/federate/solr/logic/Literal.java index edea5a1d7a..9dbaa67736 100644 --- a/source/net/yacy/cora/federate/solr/logic/Literal.java +++ b/source/net/yacy/cora/federate/solr/logic/Literal.java @@ -1,7 +1,7 @@ /** * Literal * Copyright 2014 by Michael Peter Christen - * First released 03.08.2014 at http://yacy.net + * First released 03.08.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/logic/LongLiteral.java b/source/net/yacy/cora/federate/solr/logic/LongLiteral.java index 0b6dca17f5..67677d2aee 100644 --- a/source/net/yacy/cora/federate/solr/logic/LongLiteral.java +++ b/source/net/yacy/cora/federate/solr/logic/LongLiteral.java @@ -1,7 +1,7 @@ /** * LongLiteral * Copyright 2014 by Michael Peter Christen - * First released 24.10.2014 at http://yacy.net + * First released 24.10.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/logic/Negation.java b/source/net/yacy/cora/federate/solr/logic/Negation.java index 5042342b16..add759ab71 100644 --- a/source/net/yacy/cora/federate/solr/logic/Negation.java +++ b/source/net/yacy/cora/federate/solr/logic/Negation.java @@ -1,7 +1,7 @@ /** * Negation * Copyright 2014 by Michael Peter Christen - * First released 03.08.2014 at http://yacy.net + * First released 03.08.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/logic/StringLiteral.java b/source/net/yacy/cora/federate/solr/logic/StringLiteral.java index fc00137dfe..c078f37ff9 100644 --- a/source/net/yacy/cora/federate/solr/logic/StringLiteral.java +++ b/source/net/yacy/cora/federate/solr/logic/StringLiteral.java @@ -1,7 +1,7 @@ /** * StringLiteral * Copyright 2014 by Michael Peter Christen - * First released 03.08.2014 at http://yacy.net + * First released 03.08.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/logic/Term.java b/source/net/yacy/cora/federate/solr/logic/Term.java index 685e0bf7fc..c9f208e977 100644 --- a/source/net/yacy/cora/federate/solr/logic/Term.java +++ b/source/net/yacy/cora/federate/solr/logic/Term.java @@ -1,7 +1,7 @@ /** * Term * Copyright 2014 by Michael Peter Christen - * First released 03.08.2014 at http://yacy.net + * First released 03.08.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/logic/TermTools.java b/source/net/yacy/cora/federate/solr/logic/TermTools.java index 0be80e9262..7c167a9062 100644 --- a/source/net/yacy/cora/federate/solr/logic/TermTools.java +++ b/source/net/yacy/cora/federate/solr/logic/TermTools.java @@ -1,7 +1,7 @@ /** * TermTools * Copyright 2014 by Michael Peter Christen - * First released 04.08.2014 at http://yacy.net + * First released 04.08.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java index 3504048c21..f25e599afe 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java @@ -1,7 +1,7 @@ /** * FastXMLResponseWriter * Copyright 2012 by Michael Peter Christen - * First released 06.08.2012 at http://yacy.net + * First released 06.08.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/responsewriter/FlatJSONResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/FlatJSONResponseWriter.java index 48ec35d47a..57047c9514 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/FlatJSONResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/FlatJSONResponseWriter.java @@ -1,7 +1,7 @@ /** * FlatJSONResponseWriter * Copyright 2017 by Michael Peter Christen - * First released 30.03.2017 at http://yacy.net + * First released 30.03.2017 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java index e57b056dab..d662c119cc 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java @@ -1,7 +1,7 @@ /** * GrepHTMLResponseWriter * Copyright 2013 by Michael Peter Christen - * First released 09.06.2013 at http://yacy.net + * First released 09.06.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java index 51ff9b8f1a..8e03fb832f 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java @@ -1,7 +1,7 @@ /** * HTMLResponseWriter * Copyright 2013 by Michael Peter Christen - * First released 09.06.2013 at http://yacy.net + * First released 09.06.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java index 8fed925acc..6786f0ed40 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java @@ -1,7 +1,7 @@ /** * OpensearchResponseWriter * Copyright 2012 by Michael Peter Christen - * First released 06.08.2012 at http://yacy.net + * First released 06.08.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java index 9ff2da1bc9..b7395a2844 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java @@ -1,7 +1,7 @@ /** * JsonResponseWriter * Copyright 2012 by Michael Peter Christen - * First released 10.09.2012 at http://yacy.net + * First released 10.09.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/yacy/CacheStrategy.java b/source/net/yacy/cora/federate/yacy/CacheStrategy.java index 5b43a9dd5d..addeb62fcf 100644 --- a/source/net/yacy/cora/federate/yacy/CacheStrategy.java +++ b/source/net/yacy/cora/federate/yacy/CacheStrategy.java @@ -1,7 +1,7 @@ /** * CacheStrategy * Copyright 2011 by Michael Peter Christen - * First released 2011 at http://yacy.net + * First released 2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/yacy/Distribution.java b/source/net/yacy/cora/federate/yacy/Distribution.java index a35de9acae..be4d2c6f81 100644 --- a/source/net/yacy/cora/federate/yacy/Distribution.java +++ b/source/net/yacy/cora/federate/yacy/Distribution.java @@ -1,7 +1,7 @@ /** * VerticalPartition * Copyright 2009 by Michael Peter Christen - * First released 28.01.2009 at http://yacy.net + * First released 28.01.2009 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/yacy/Peer.java b/source/net/yacy/cora/federate/yacy/Peer.java index 457288dc2c..608dc0031d 100644 --- a/source/net/yacy/cora/federate/yacy/Peer.java +++ b/source/net/yacy/cora/federate/yacy/Peer.java @@ -1,7 +1,7 @@ /** * Peer * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 21.09.2012 at http://yacy.net + * First released 21.09.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/yacy/Peers.java b/source/net/yacy/cora/federate/yacy/Peers.java index 4903131897..e982fd8272 100644 --- a/source/net/yacy/cora/federate/yacy/Peers.java +++ b/source/net/yacy/cora/federate/yacy/Peers.java @@ -1,7 +1,7 @@ /** * Peers * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 21.09.2012 at http://yacy.net + * First released 21.09.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/federate/yacy/api/Network.java b/source/net/yacy/cora/federate/yacy/api/Network.java index 486b1b917f..0b83167195 100644 --- a/source/net/yacy/cora/federate/yacy/api/Network.java +++ b/source/net/yacy/cora/federate/yacy/api/Network.java @@ -1,7 +1,7 @@ /** * Peers * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 21.09.2012 at http://yacy.net + * First released 21.09.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/language/phonetic/Phonetic.java b/source/net/yacy/cora/language/phonetic/Phonetic.java index c7d6ff2afa..11c468e518 100644 --- a/source/net/yacy/cora/language/phonetic/Phonetic.java +++ b/source/net/yacy/cora/language/phonetic/Phonetic.java @@ -1,7 +1,7 @@ /** * Phonetic * Copyright 201 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 13.12.2011 at http://yacy.net + * First released 13.12.2011 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/lod/Literal.java b/source/net/yacy/cora/lod/Literal.java index 94b0bea70b..40e3acb1ef 100644 --- a/source/net/yacy/cora/lod/Literal.java +++ b/source/net/yacy/cora/lod/Literal.java @@ -1,7 +1,7 @@ /** * Literal * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 18.12.2011 at http://yacy.net + * First released 18.12.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/cora/lod/Node.java b/source/net/yacy/cora/lod/Node.java index c2f279b708..a33a1571ba 100644 --- a/source/net/yacy/cora/lod/Node.java +++ b/source/net/yacy/cora/lod/Node.java @@ -1,7 +1,7 @@ /** * AbstractScoreMap * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 16.12.2011 at http://yacy.net + * First released 16.12.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/cora/lod/Syntax.java b/source/net/yacy/cora/lod/Syntax.java index b34a78e872..084dbf1a4b 100644 --- a/source/net/yacy/cora/lod/Syntax.java +++ b/source/net/yacy/cora/lod/Syntax.java @@ -1,7 +1,7 @@ /** * Syntax * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 17.12.2011 at http://yacy.net + * First released 17.12.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/cora/lod/Vocabulary.java b/source/net/yacy/cora/lod/Vocabulary.java index 3566128e39..dde7d605d4 100644 --- a/source/net/yacy/cora/lod/Vocabulary.java +++ b/source/net/yacy/cora/lod/Vocabulary.java @@ -1,7 +1,7 @@ /** * Vocabulary * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 16.12.2011 at http://yacy.net + * First released 16.12.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/cora/lod/vocabulary/AnnoteaA.java b/source/net/yacy/cora/lod/vocabulary/AnnoteaA.java index eb0e2fef27..139375e442 100644 --- a/source/net/yacy/cora/lod/vocabulary/AnnoteaA.java +++ b/source/net/yacy/cora/lod/vocabulary/AnnoteaA.java @@ -1,7 +1,7 @@ /** * AnnoteaA * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 16.12.2011 at http://yacy.net + * First released 16.12.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/lod/vocabulary/AnnoteaB.java b/source/net/yacy/cora/lod/vocabulary/AnnoteaB.java index 3591c6d562..fc16071f5c 100644 --- a/source/net/yacy/cora/lod/vocabulary/AnnoteaB.java +++ b/source/net/yacy/cora/lod/vocabulary/AnnoteaB.java @@ -1,7 +1,7 @@ /** * AnnoteaB * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 16.12.2011 at http://yacy.net + * First released 16.12.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/lod/vocabulary/CreativeCommons.java b/source/net/yacy/cora/lod/vocabulary/CreativeCommons.java index d21e11bf22..390fd45036 100644 --- a/source/net/yacy/cora/lod/vocabulary/CreativeCommons.java +++ b/source/net/yacy/cora/lod/vocabulary/CreativeCommons.java @@ -1,7 +1,7 @@ /** * CreativeCommons * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 17.12.2011 at http://yacy.net + * First released 17.12.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/cora/lod/vocabulary/DMOZ.java b/source/net/yacy/cora/lod/vocabulary/DMOZ.java index 74080a26c7..f4b290633c 100644 --- a/source/net/yacy/cora/lod/vocabulary/DMOZ.java +++ b/source/net/yacy/cora/lod/vocabulary/DMOZ.java @@ -1,7 +1,7 @@ /** * DMOZ * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 16.12.2011 at http://yacy.net + * First released 16.12.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/lod/vocabulary/DublinCore.java b/source/net/yacy/cora/lod/vocabulary/DublinCore.java index 0f8a54401f..c12a9ecf82 100644 --- a/source/net/yacy/cora/lod/vocabulary/DublinCore.java +++ b/source/net/yacy/cora/lod/vocabulary/DublinCore.java @@ -1,7 +1,7 @@ /** * DublinCore * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 16.12.2011 at http://yacy.net + * First released 16.12.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/cora/lod/vocabulary/Foaf.java b/source/net/yacy/cora/lod/vocabulary/Foaf.java index 760c969e19..c4b2000f54 100644 --- a/source/net/yacy/cora/lod/vocabulary/Foaf.java +++ b/source/net/yacy/cora/lod/vocabulary/Foaf.java @@ -1,7 +1,7 @@ /** * Foaf * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 17.12.2011 at http://yacy.net + * First released 17.12.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/cora/lod/vocabulary/Geo.java b/source/net/yacy/cora/lod/vocabulary/Geo.java index 1b6084bbb9..ed61b37fc6 100644 --- a/source/net/yacy/cora/lod/vocabulary/Geo.java +++ b/source/net/yacy/cora/lod/vocabulary/Geo.java @@ -1,7 +1,7 @@ /** * Geo * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 16.12.2011 at http://yacy.net + * First released 16.12.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/cora/lod/vocabulary/HttpHeader.java b/source/net/yacy/cora/lod/vocabulary/HttpHeader.java index ae69342958..341e1ec711 100644 --- a/source/net/yacy/cora/lod/vocabulary/HttpHeader.java +++ b/source/net/yacy/cora/lod/vocabulary/HttpHeader.java @@ -1,7 +1,7 @@ /** * HttpHeader * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 16.12.2011 at http://yacy.net + * First released 16.12.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/cora/lod/vocabulary/Owl.java b/source/net/yacy/cora/lod/vocabulary/Owl.java index a4c56e5e09..478b0f5476 100644 --- a/source/net/yacy/cora/lod/vocabulary/Owl.java +++ b/source/net/yacy/cora/lod/vocabulary/Owl.java @@ -1,7 +1,7 @@ /** * Owl * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 11.06.2011 at http://yacy.net + * First released 11.06.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/lod/vocabulary/Rdf.java b/source/net/yacy/cora/lod/vocabulary/Rdf.java index 9b394d5d4a..9e784da7be 100644 --- a/source/net/yacy/cora/lod/vocabulary/Rdf.java +++ b/source/net/yacy/cora/lod/vocabulary/Rdf.java @@ -1,7 +1,7 @@ /** * Rdf * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 17.12.2011 at http://yacy.net + * First released 17.12.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/cora/lod/vocabulary/YaCyMetadata.java b/source/net/yacy/cora/lod/vocabulary/YaCyMetadata.java index 2b8ecde4d9..691cf8b0c8 100644 --- a/source/net/yacy/cora/lod/vocabulary/YaCyMetadata.java +++ b/source/net/yacy/cora/lod/vocabulary/YaCyMetadata.java @@ -1,7 +1,7 @@ /** * YaCyMetadata * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 16.12.2011 at http://yacy.net + * First released 16.12.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/cora/order/AbstractOrder.java b/source/net/yacy/cora/order/AbstractOrder.java index 42e38f0cde..447c7d0ed0 100644 --- a/source/net/yacy/cora/order/AbstractOrder.java +++ b/source/net/yacy/cora/order/AbstractOrder.java @@ -1,7 +1,7 @@ /** * AbstractOrder * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 25.08.2011 at http://yacy.net + * First released 25.08.2011 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/order/Order.java b/source/net/yacy/cora/order/Order.java index 4872f27841..9c2fe23863 100644 --- a/source/net/yacy/cora/order/Order.java +++ b/source/net/yacy/cora/order/Order.java @@ -1,7 +1,7 @@ /** * Order * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 25.08.2011 at http://yacy.net + * First released 25.08.2011 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/order/RatingOrder.java b/source/net/yacy/cora/order/RatingOrder.java index 95b6cb64cf..de4673f308 100644 --- a/source/net/yacy/cora/order/RatingOrder.java +++ b/source/net/yacy/cora/order/RatingOrder.java @@ -1,7 +1,7 @@ /** * RatingOrder.java * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 25.08.2011 at http://yacy.net + * First released 25.08.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/cora/plugin/ClassProvider.java b/source/net/yacy/cora/plugin/ClassProvider.java index a8b02a5358..4beafdd290 100644 --- a/source/net/yacy/cora/plugin/ClassProvider.java +++ b/source/net/yacy/cora/plugin/ClassProvider.java @@ -1,7 +1,7 @@ /** * ClassProvider * Copyright 201 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 13.12.2011 at http://yacy.net + * First released 13.12.2011 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/protocol/ByteArrayBody.java b/source/net/yacy/cora/protocol/ByteArrayBody.java index 132b20f415..f65829ca30 100644 --- a/source/net/yacy/cora/protocol/ByteArrayBody.java +++ b/source/net/yacy/cora/protocol/ByteArrayBody.java @@ -1,7 +1,7 @@ /** * ByteArrayBody * Copyright 2010 by Sebastian Gaebel - * First released 01.07.2010 at http://yacy.net + * First released 01.07.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/protocol/ClientIdentification.java b/source/net/yacy/cora/protocol/ClientIdentification.java index 761b655516..8e3eba44bc 100644 --- a/source/net/yacy/cora/protocol/ClientIdentification.java +++ b/source/net/yacy/cora/protocol/ClientIdentification.java @@ -1,7 +1,7 @@ /** * ClientIdentification * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 26.04.2011 at http://yacy.net + * First released 26.04.2011 at https://yacy.net * * $LastChangedDate: 2011-04-21 23:59:56 +0200 (Do, 21 Apr 2011) $ * $LastChangedRevision: 7673 $ @@ -11,12 +11,12 @@ * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -34,7 +34,7 @@ public class ClientIdentification { public static final int clientTimeoutInit = 10000; public static final int minimumLocalDeltaInit = 10; // the minimum time difference between access of the same local domain public static final int minimumGlobalDeltaInit = 250; // the minimum time difference between access of the same global domain - + public static class Agent { public final String userAgent; // the name that is send in http request to identify the agent public final String[] robotIDs; // the name that is used in robots.txt to identify the agent @@ -47,7 +47,7 @@ public Agent(final String userAgent, final String[] robotIDs, final int minimumD this.clientTimeout = clientTimeout; } } - + private final static String[] browserAgents = new String[]{ // fake browser user agents are NOT AVAILABLE IN P2P OPERATION, only on special customer configurations (commercial users demanded this, I personally think this is inadvisable) "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:22.0) Gecko/20100101 Firefox/22.0", @@ -66,7 +66,7 @@ public Agent(final String userAgent, final String[] robotIDs, final int minimumD "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:21.0) Gecko/20100101 Firefox/21.0" }; private static final Random random = new Random(System.currentTimeMillis()); - private static Map agents = new ConcurrentHashMap(); + private static Map agents = new ConcurrentHashMap<>(); public final static String yacyInternetCrawlerAgentName = "YaCy Internet (cautious)"; public static Agent yacyInternetCrawlerAgent = null; // defined later in static public final static String yacyIntranetCrawlerAgentName = "YaCy Intranet (greedy)"; @@ -85,7 +85,7 @@ public Agent(final String userAgent, final String[] robotIDs, final int minimumD public static final String yacySystem = System.getProperty("os.arch", "no-os-arch") + " " + System.getProperty("os.name", "no-os-name") + " " + System.getProperty("os.version", "no-os-version") + "; " + "java " + System.getProperty("java.version", "no-java-version") + "; " + generateLocation(); // keep this before the following static initialization block as this constant is used by generateYaCyBot() - + static { generateYaCyBot("new"); browserAgent = new Agent(browserAgents[random.nextInt(browserAgents.length)], new String[]{"Mozilla"}, minimumLocalDeltaInit, clientTimeoutInit); @@ -93,23 +93,23 @@ public Agent(final String userAgent, final String[] robotIDs, final int minimumD agents.put(browserAgentName, browserAgent); agents.put(yacyProxyAgentName, yacyProxyAgent); } - + /** * produce a YaCy user agent string * @param addinfo * @return */ - public static void generateYaCyBot(String addinfo) { - String agentString = "yacybot (" + addinfo + "; " + yacySystem + ") http://yacy.net/bot.html"; + public static void generateYaCyBot(final String addinfo) { + final String agentString = "yacybot (" + addinfo + "; " + yacySystem + ") https://yacy.net/bot.html"; yacyInternetCrawlerAgent = new Agent(agentString, new String[]{"yacybot"}, minimumGlobalDeltaInit, clientTimeoutInit); yacyIntranetCrawlerAgent = new Agent(agentString, new String[]{"yacybot"}, minimumLocalDeltaInit, clientTimeoutInit); // must have the same userAgent String as the web crawler because this is also used for snippets agents.put(yacyInternetCrawlerAgentName, yacyInternetCrawlerAgent); agents.put(yacyIntranetCrawlerAgentName, yacyIntranetCrawlerAgent); } - - public static void generateCustomBot(String name, String string, int minimumdelta, int clienttimeout) { + + public static void generateCustomBot(final String name, final String string, final int minimumdelta, final int clienttimeout) { if (name.toLowerCase().indexOf("yacy") >= 0 || string.toLowerCase().indexOf("yacy") >= 0) return; // don't allow 'yacy' in custom bot strings - String agentString = string.replace("$$SYSTEM$$", yacySystem.replace("java", "O")); + final String agentString = string.replace("$$SYSTEM$$", yacySystem.replace("java", "O")); agents.put(customAgentName, new Agent(agentString, new String[]{name}, minimumdelta, clienttimeout)); } @@ -117,15 +117,15 @@ public static void generateCustomBot(String name, String string, int minimumdelt * get the default agent * @param newagent */ - public static Agent getAgent(String agentName) { + public static Agent getAgent(final String agentName) { if (agentName == null || agentName.length() == 0) return yacyInternetCrawlerAgent; - Agent agent = agents.get(agentName); + final Agent agent = agents.get(agentName); return agent == null ? yacyInternetCrawlerAgent : agent; } - + /** * generating the location string - * + * * @return */ public static String generateLocation() { @@ -140,9 +140,9 @@ public static String generateLocation() { /** * gets the location out of the user agent - * + * * location must be after last ; and before first ) - * + * * @param userAgent in form "useragentinfo (some params; _location_) additional info" * @return */ diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java index 6bb6c63ef9..527d6cf670 100644 --- a/source/net/yacy/cora/protocol/Domains.java +++ b/source/net/yacy/cora/protocol/Domains.java @@ -1,7 +1,7 @@ /** * Domains * Copyright 2007 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 23.7.2007 at http://yacy.net + * First released 23.7.2007 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/protocol/HeaderFramework.java b/source/net/yacy/cora/protocol/HeaderFramework.java index 0eeb3ddf24..5eba1cbbfc 100644 --- a/source/net/yacy/cora/protocol/HeaderFramework.java +++ b/source/net/yacy/cora/protocol/HeaderFramework.java @@ -1,7 +1,7 @@ /** * HeaderFramework * Copyright 2004 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 2004 at http://yacy.net + * First released 2004 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/protocol/RequestHeader.java b/source/net/yacy/cora/protocol/RequestHeader.java index df1a09a1a6..ccc8be8086 100644 --- a/source/net/yacy/cora/protocol/RequestHeader.java +++ b/source/net/yacy/cora/protocol/RequestHeader.java @@ -1,7 +1,7 @@ /** * RequestHeader * Copyright 2008 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 22.08.2008 at http://yacy.net + * First released 22.08.2008 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/protocol/ResponseHeader.java b/source/net/yacy/cora/protocol/ResponseHeader.java index 9ae002d136..c7be30fa73 100644 --- a/source/net/yacy/cora/protocol/ResponseHeader.java +++ b/source/net/yacy/cora/protocol/ResponseHeader.java @@ -1,7 +1,7 @@ /** * ResponseHeader * Copyright 2008 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 22.08.2008 at http://yacy.net + * First released 22.08.2008 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/protocol/Scanner.java b/source/net/yacy/cora/protocol/Scanner.java index 9af8c88416..a034e0fa9f 100644 --- a/source/net/yacy/cora/protocol/Scanner.java +++ b/source/net/yacy/cora/protocol/Scanner.java @@ -1,7 +1,7 @@ /** * Scanner * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 28.10.2010 at http://yacy.net + * First released 28.10.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/protocol/TimeoutRequest.java b/source/net/yacy/cora/protocol/TimeoutRequest.java index b4485fa0ce..9e7fa2f9de 100644 --- a/source/net/yacy/cora/protocol/TimeoutRequest.java +++ b/source/net/yacy/cora/protocol/TimeoutRequest.java @@ -1,7 +1,7 @@ /** * TimeoutRequest * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 08.10.2007 at http://yacy.net + * First released 08.10.2007 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/protocol/http/GzipCompressingEntity.java b/source/net/yacy/cora/protocol/http/GzipCompressingEntity.java index e2649239c7..ead13772ff 100644 --- a/source/net/yacy/cora/protocol/http/GzipCompressingEntity.java +++ b/source/net/yacy/cora/protocol/http/GzipCompressingEntity.java @@ -1,7 +1,7 @@ /** * GzipCompressingEntity * Copyright 2010 by Sebastian Gaebel - * First released 01.07.2010 at http://yacy.net + * First released 01.07.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/protocol/http/GzipRequestInterceptor.java b/source/net/yacy/cora/protocol/http/GzipRequestInterceptor.java index 87160b8fe6..0d7d07c335 100644 --- a/source/net/yacy/cora/protocol/http/GzipRequestInterceptor.java +++ b/source/net/yacy/cora/protocol/http/GzipRequestInterceptor.java @@ -1,7 +1,7 @@ /** * GzipRequestInterceptor * Copyright 2010 by Sebastian Gaebel - * First released 01.07.2010 at http://yacy.net + * First released 01.07.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/protocol/http/GzipResponseInterceptor.java b/source/net/yacy/cora/protocol/http/GzipResponseInterceptor.java index 5290239f9e..4b91f20aad 100644 --- a/source/net/yacy/cora/protocol/http/GzipResponseInterceptor.java +++ b/source/net/yacy/cora/protocol/http/GzipResponseInterceptor.java @@ -1,7 +1,7 @@ /** * GzipResponseInterceptor * Copyright 2010 by Sebastian Gaebel - * First released 01.07.2010 at http://yacy.net + * First released 01.07.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java index 45e206b6da..f924fd1d21 100644 --- a/source/net/yacy/cora/protocol/http/HTTPClient.java +++ b/source/net/yacy/cora/protocol/http/HTTPClient.java @@ -1,7 +1,7 @@ /** * HTTPClient * Copyright 2010 by Sebastian Gaebel - * First released 01.07.2010 at http://yacy.net + * First released 01.07.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/protocol/http/LinkExtractor.java b/source/net/yacy/cora/protocol/http/LinkExtractor.java index 7935492864..c73ce54196 100644 --- a/source/net/yacy/cora/protocol/http/LinkExtractor.java +++ b/source/net/yacy/cora/protocol/http/LinkExtractor.java @@ -1,7 +1,7 @@ /** * LinkExtractor * Copyright 2011 by Michael Peter Christen - * First released 2.01.2011 at http://yacy.net + * First released 2.01.2011 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/protocol/http/ProxySettings.java b/source/net/yacy/cora/protocol/http/ProxySettings.java index 710ab12582..7c25b72d56 100644 --- a/source/net/yacy/cora/protocol/http/ProxySettings.java +++ b/source/net/yacy/cora/protocol/http/ProxySettings.java @@ -1,7 +1,7 @@ /** * ProxySettings * Copyright 2010 by Michael Peter Christen - * First released 25.05.2010 at http://yacy.net + * First released 25.05.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/sorting/AbstractScoreMap.java b/source/net/yacy/cora/sorting/AbstractScoreMap.java index b662925a14..b1412aea4a 100644 --- a/source/net/yacy/cora/sorting/AbstractScoreMap.java +++ b/source/net/yacy/cora/sorting/AbstractScoreMap.java @@ -1,7 +1,7 @@ /** * AbstractScoreMap * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 28.04.2011 at http://yacy.net + * First released 28.04.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/cora/sorting/Array.java b/source/net/yacy/cora/sorting/Array.java index 3cb2b72848..a7ce775b97 100644 --- a/source/net/yacy/cora/sorting/Array.java +++ b/source/net/yacy/cora/sorting/Array.java @@ -1,7 +1,7 @@ /** * Array * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 16.07.2011 at http://yacy.net + * First released 16.07.2011 at https://yacy.net * * $LastChangedDate: 2011-05-30 10:53:58 +0200 (Mo, 30 Mai 2011) $ * $LastChangedRevision: 7759 $ diff --git a/source/net/yacy/cora/sorting/ClusteredScoreMap.java b/source/net/yacy/cora/sorting/ClusteredScoreMap.java index fb55cda4aa..831bbe8e6e 100644 --- a/source/net/yacy/cora/sorting/ClusteredScoreMap.java +++ b/source/net/yacy/cora/sorting/ClusteredScoreMap.java @@ -1,7 +1,7 @@ /** * ScoreCluster * Copyright 2004, 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 28.09.2004 at http://yacy.net + * First released 28.09.2004 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/sorting/ConcurrentScoreMap.java b/source/net/yacy/cora/sorting/ConcurrentScoreMap.java index 716e684265..1b9b36d3b6 100644 --- a/source/net/yacy/cora/sorting/ConcurrentScoreMap.java +++ b/source/net/yacy/cora/sorting/ConcurrentScoreMap.java @@ -1,7 +1,7 @@ /** * ConcurrentScoreMap * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 13.03.2011 at http://yacy.net + * First released 13.03.2011 at https://yacy.net * * $LastChangedDate: 2011-03-08 02:51:51 +0100 (Di, 08 Mrz 2011) $ * $LastChangedRevision: 7567 $ diff --git a/source/net/yacy/cora/sorting/OrderedScoreMap.java b/source/net/yacy/cora/sorting/OrderedScoreMap.java index 9b9453423d..dce21af488 100644 --- a/source/net/yacy/cora/sorting/OrderedScoreMap.java +++ b/source/net/yacy/cora/sorting/OrderedScoreMap.java @@ -1,7 +1,7 @@ /** * ScoreMap * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 14.10.2010 at http://yacy.net + * First released 14.10.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/sorting/Rating.java b/source/net/yacy/cora/sorting/Rating.java index 9c24427ca6..78388a3c1a 100644 --- a/source/net/yacy/cora/sorting/Rating.java +++ b/source/net/yacy/cora/sorting/Rating.java @@ -1,7 +1,7 @@ /** * Rating * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 25.08.2011 at http://yacy.net + * First released 25.08.2011 at https://yacy.net * * $LastChangedDate: 2011-03-08 02:51:51 +0100 (Di, 08 Mrz 2011) $ * $LastChangedRevision: 7567 $ diff --git a/source/net/yacy/cora/sorting/ReversibleScoreMap.java b/source/net/yacy/cora/sorting/ReversibleScoreMap.java index 09e2fe74a5..9185af0d27 100644 --- a/source/net/yacy/cora/sorting/ReversibleScoreMap.java +++ b/source/net/yacy/cora/sorting/ReversibleScoreMap.java @@ -1,7 +1,7 @@ /** * ReversibleScoreMap * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 14.10.2010 at http://yacy.net + * First released 14.10.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/sorting/ScoreComparator.java b/source/net/yacy/cora/sorting/ScoreComparator.java index 23919e443d..bec863f204 100644 --- a/source/net/yacy/cora/sorting/ScoreComparator.java +++ b/source/net/yacy/cora/sorting/ScoreComparator.java @@ -1,7 +1,7 @@ /** * ScoreComparator * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 23.08.2011 at http://yacy.net + * First released 23.08.2011 at https://yacy.net * * $LastChangedDate: 2011-03-08 02:51:51 +0100 (Di, 08 Mrz 2011) $ * $LastChangedRevision: 7567 $ diff --git a/source/net/yacy/cora/sorting/ScoreMap.java b/source/net/yacy/cora/sorting/ScoreMap.java index 52d3a74073..1755e3de28 100644 --- a/source/net/yacy/cora/sorting/ScoreMap.java +++ b/source/net/yacy/cora/sorting/ScoreMap.java @@ -1,7 +1,7 @@ /** * ScoreMap * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 14.10.2010 at http://yacy.net + * First released 14.10.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/sorting/Sortable.java b/source/net/yacy/cora/sorting/Sortable.java index d32adc12a7..6f0834732d 100644 --- a/source/net/yacy/cora/sorting/Sortable.java +++ b/source/net/yacy/cora/sorting/Sortable.java @@ -1,7 +1,7 @@ /** * Sortable * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 16.07.2011 at http://yacy.net + * First released 16.07.2011 at https://yacy.net * * $LastChangedDate: 2011-05-30 10:53:58 +0200 (Mo, 30 Mai 2011) $ * $LastChangedRevision: 7759 $ diff --git a/source/net/yacy/cora/sorting/WeakPriorityBlockingQueue.java b/source/net/yacy/cora/sorting/WeakPriorityBlockingQueue.java index 6edf6bfb8f..7be4378a92 100644 --- a/source/net/yacy/cora/sorting/WeakPriorityBlockingQueue.java +++ b/source/net/yacy/cora/sorting/WeakPriorityBlockingQueue.java @@ -2,7 +2,7 @@ * WeakPriorityBlockingQueue * an priority blocking queue that drains elements if it gets too large * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 09.09.2010 at http://yacy.net + * First released 09.09.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/storage/ARC.java b/source/net/yacy/cora/storage/ARC.java index 8cca976ae2..b532168153 100644 --- a/source/net/yacy/cora/storage/ARC.java +++ b/source/net/yacy/cora/storage/ARC.java @@ -2,7 +2,7 @@ * ARC * an interface for Adaptive Replacement Caches * Copyright 2009 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 29.08.2009 at http://yacy.net + * First released 29.08.2009 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/storage/ARH.java b/source/net/yacy/cora/storage/ARH.java index 1532f011e2..160033269a 100644 --- a/source/net/yacy/cora/storage/ARH.java +++ b/source/net/yacy/cora/storage/ARH.java @@ -2,7 +2,7 @@ * ARH * an interface for Adaptive Replacement Handles * Copyright 2014 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 15.01.2014 at http://yacy.net + * First released 15.01.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/storage/AbstractMapStore.java b/source/net/yacy/cora/storage/AbstractMapStore.java index 899a472add..c9c5cd9669 100644 --- a/source/net/yacy/cora/storage/AbstractMapStore.java +++ b/source/net/yacy/cora/storage/AbstractMapStore.java @@ -1,7 +1,7 @@ /** * AbstractMapStore * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 16.12.2011 at http://yacy.net + * First released 16.12.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/cora/storage/ComparableARC.java b/source/net/yacy/cora/storage/ComparableARC.java index 24ed59effc..17f666a72b 100644 --- a/source/net/yacy/cora/storage/ComparableARC.java +++ b/source/net/yacy/cora/storage/ComparableARC.java @@ -2,7 +2,7 @@ * ComparableARC * an Adaptive Replacement Cache for comparable objects * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 24.08.2010 at http://yacy.net + * First released 24.08.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/storage/ConcurrentARC.java b/source/net/yacy/cora/storage/ConcurrentARC.java index 14786bb6ef..1e1ee82376 100644 --- a/source/net/yacy/cora/storage/ConcurrentARC.java +++ b/source/net/yacy/cora/storage/ConcurrentARC.java @@ -1,7 +1,7 @@ /** * ConcurrentARC * Copyright 2009 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 17.04.2009 at http://yacy.net + * First released 17.04.2009 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/storage/ConcurrentARH.java b/source/net/yacy/cora/storage/ConcurrentARH.java index 6225869fd2..e7b5d1b99a 100644 --- a/source/net/yacy/cora/storage/ConcurrentARH.java +++ b/source/net/yacy/cora/storage/ConcurrentARH.java @@ -2,7 +2,7 @@ * ConcurrentARH * an interface for Adaptive Replacement Handles * Copyright 2014 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 15.01.2014 at http://yacy.net + * First released 15.01.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/storage/Configuration.java b/source/net/yacy/cora/storage/Configuration.java index 64dc3cd6dd..787c5677cb 100644 --- a/source/net/yacy/cora/storage/Configuration.java +++ b/source/net/yacy/cora/storage/Configuration.java @@ -1,7 +1,7 @@ /** * Configuration * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 29.06.2011 at http://yacy.net + * First released 29.06.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/storage/Files.java b/source/net/yacy/cora/storage/Files.java index 89f5e614e1..0be7b1d62e 100644 --- a/source/net/yacy/cora/storage/Files.java +++ b/source/net/yacy/cora/storage/Files.java @@ -1,7 +1,7 @@ /** * Files * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 29.06.2011 at http://yacy.net + * First released 29.06.2011 at https://yacy.net * * $LastChangedDate: 2011-05-30 10:53:58 +0200 (Mo, 30 Mai 2011) $ * $LastChangedRevision: 7759 $ diff --git a/source/net/yacy/cora/storage/HashARC.java b/source/net/yacy/cora/storage/HashARC.java index 3a9f54dd68..d33261b7cb 100644 --- a/source/net/yacy/cora/storage/HashARC.java +++ b/source/net/yacy/cora/storage/HashARC.java @@ -2,7 +2,7 @@ * HashARC * an Adaptive Replacement Cache for objects that can be compared using hashing * Copyright 2009 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 17.04.2009 at http://yacy.net + * First released 17.04.2009 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/storage/KeyList.java b/source/net/yacy/cora/storage/KeyList.java index 563056b49e..0112183751 100644 --- a/source/net/yacy/cora/storage/KeyList.java +++ b/source/net/yacy/cora/storage/KeyList.java @@ -1,7 +1,7 @@ /** * KeyList * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 18.4.2011 at http://yacy.net + * First released 18.4.2011 at https://yacy.net * * $LastChangedDate: 2011-03-22 10:34:10 +0100 (Di, 22 Mrz 2011) $ * $LastChangedRevision: 7619 $ diff --git a/source/net/yacy/cora/storage/MapStore.java b/source/net/yacy/cora/storage/MapStore.java index 5a051b40e1..f1cdac9fac 100644 --- a/source/net/yacy/cora/storage/MapStore.java +++ b/source/net/yacy/cora/storage/MapStore.java @@ -1,7 +1,7 @@ /** * MapStore * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 16.12.2011 at http://yacy.net + * First released 16.12.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/cora/storage/OutOfLimitsException.java b/source/net/yacy/cora/storage/OutOfLimitsException.java index bc102bcce4..73c40141fe 100644 --- a/source/net/yacy/cora/storage/OutOfLimitsException.java +++ b/source/net/yacy/cora/storage/OutOfLimitsException.java @@ -1,7 +1,7 @@ /** * OutOfLimitsException * Copyright 2006 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 17.01.2006 at http://yacy.net + * First released 17.01.2006 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/storage/SimpleARC.java b/source/net/yacy/cora/storage/SimpleARC.java index 4977b1f108..0630b824f2 100644 --- a/source/net/yacy/cora/storage/SimpleARC.java +++ b/source/net/yacy/cora/storage/SimpleARC.java @@ -2,7 +2,7 @@ * SimpleARC * a Simple Adaptive Replacement Cache * Copyright 2009 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 17.04.2009 at http://yacy.net + * First released 17.04.2009 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/cora/storage/SizeLimitedMap.java b/source/net/yacy/cora/storage/SizeLimitedMap.java index 0d15d7a776..bb66737a67 100644 --- a/source/net/yacy/cora/storage/SizeLimitedMap.java +++ b/source/net/yacy/cora/storage/SizeLimitedMap.java @@ -1,7 +1,7 @@ /** * SizeLimitedMap * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 04.07.2012 at http://yacy.net + * First released 04.07.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/storage/SizeLimitedSet.java b/source/net/yacy/cora/storage/SizeLimitedSet.java index ecb3701b99..d45497860c 100644 --- a/source/net/yacy/cora/storage/SizeLimitedSet.java +++ b/source/net/yacy/cora/storage/SizeLimitedSet.java @@ -1,7 +1,7 @@ /** * SizeLimitedSet * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 04.07.2012 at http://yacy.net + * First released 04.07.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/storage/ZIPReader.java b/source/net/yacy/cora/storage/ZIPReader.java index 8062be5330..2f3bcce348 100644 --- a/source/net/yacy/cora/storage/ZIPReader.java +++ b/source/net/yacy/cora/storage/ZIPReader.java @@ -1,7 +1,7 @@ /** * ZIPReader * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 24.09.2012 at http://yacy.net + * First released 24.09.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/storage/ZIPWriter.java b/source/net/yacy/cora/storage/ZIPWriter.java index 93cc1b69fb..47d964b3b7 100644 --- a/source/net/yacy/cora/storage/ZIPWriter.java +++ b/source/net/yacy/cora/storage/ZIPWriter.java @@ -1,7 +1,7 @@ /** * ZIPWriter * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 24.09.2012 at http://yacy.net + * First released 24.09.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/util/NumberTools.java b/source/net/yacy/cora/util/NumberTools.java index 454e1d094c..03799ac9c7 100644 --- a/source/net/yacy/cora/util/NumberTools.java +++ b/source/net/yacy/cora/util/NumberTools.java @@ -1,7 +1,7 @@ /** * NumberTools * Copyright 2012 by Sebastian Gaebel - * First released 21.05.2012 at http://yacy.net + * First released 21.05.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/cora/util/StringBuilderComparator.java b/source/net/yacy/cora/util/StringBuilderComparator.java index d82bc5dfe4..f2d6eec18d 100644 --- a/source/net/yacy/cora/util/StringBuilderComparator.java +++ b/source/net/yacy/cora/util/StringBuilderComparator.java @@ -1,7 +1,7 @@ /** * StringBuilderComparator.java * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 09.11.2011 at http://yacy.net + * First released 09.11.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/crawler/Balancer.java b/source/net/yacy/crawler/Balancer.java index 9d106ad46a..d68ab8994f 100644 --- a/source/net/yacy/crawler/Balancer.java +++ b/source/net/yacy/crawler/Balancer.java @@ -3,7 +3,7 @@ * SPDX-FileCopyrightText: 2014 Michael Peter Christen * SPDX-License-Identifier: GPL-2.0-or-later * Frankfurt am Main, Germany - * First released 14.04.2014 at http://yacy.net + * First released 14.04.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/crawler/HarvestProcess.java b/source/net/yacy/crawler/HarvestProcess.java index ed1270825b..6f8f247f4d 100644 --- a/source/net/yacy/crawler/HarvestProcess.java +++ b/source/net/yacy/crawler/HarvestProcess.java @@ -2,7 +2,7 @@ * HarvestProcess * SPDX-FileCopyrightText: 2012 Michael Peter Christen * SPDX-License-Identifier: GPL-2.0-or-later - * First released 06.12.2012 at http://yacy.net + * First released 06.12.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/crawler/HostBalancer.java b/source/net/yacy/crawler/HostBalancer.java index a798206f9d..c9b8133cf9 100644 --- a/source/net/yacy/crawler/HostBalancer.java +++ b/source/net/yacy/crawler/HostBalancer.java @@ -2,7 +2,7 @@ * HostQueues * SPDX-FileCopyrightText: 2013 Michael Peter Christen * SPDX-License-Identifier: GPL-2.0-or-later - * First released 24.09.2013 at http://yacy.net + * First released 24.09.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/crawler/HostQueue.java b/source/net/yacy/crawler/HostQueue.java index 893b2ce2be..cb9135400b 100644 --- a/source/net/yacy/crawler/HostQueue.java +++ b/source/net/yacy/crawler/HostQueue.java @@ -2,7 +2,7 @@ * HostQueue * SPDX-FileCopyrightText: 2013 Michael Peter Christen * SPDX-License-Identifier: GPL-2.0-or-later - * First released 24.09.2013 at http://yacy.net + * First released 24.09.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/crawler/RecrawlBusyThread.java b/source/net/yacy/crawler/RecrawlBusyThread.java index 11b3d3a981..3baa562552 100644 --- a/source/net/yacy/crawler/RecrawlBusyThread.java +++ b/source/net/yacy/crawler/RecrawlBusyThread.java @@ -2,7 +2,7 @@ * RecrawlBusyThread.java * SPDX-FileCopyrightText: 2015 by Burkhard Buelte * SPDX-License-Identifier: GPL-2.0-or-later - * First released 15.05.2015 at http://yacy.net + * First released 15.05.2015 at https://yacy.net * * This is a part of YaCy, a peer-to-peer based web search engine * diff --git a/source/net/yacy/crawler/data/Snapshots.java b/source/net/yacy/crawler/data/Snapshots.java index f5dd411a88..0463d0d429 100644 --- a/source/net/yacy/crawler/data/Snapshots.java +++ b/source/net/yacy/crawler/data/Snapshots.java @@ -2,7 +2,7 @@ * DocumentImage * SPDX-FileCopyrightText: 2014 Michael Peter Christen * SPDX-License-Identifier: GPL-2.0-or-later - * First released 29.11.2014 at http://yacy.net + * First released 29.11.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/crawler/data/Transactions.java b/source/net/yacy/crawler/data/Transactions.java index 14969b852d..499dae7dab 100644 --- a/source/net/yacy/crawler/data/Transactions.java +++ b/source/net/yacy/crawler/data/Transactions.java @@ -2,7 +2,7 @@ * Transactions * SPDX-FileCopyrightText: 2014 Michael Peter Christen * SPDX-License-Identifier: GPL-2.0-or-later - * First released 08.12.2014 at http://yacy.net + * First released 08.12.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/crawler/retrieval/FileLoader.java b/source/net/yacy/crawler/retrieval/FileLoader.java index be5568fb6b..010ffa8550 100644 --- a/source/net/yacy/crawler/retrieval/FileLoader.java +++ b/source/net/yacy/crawler/retrieval/FileLoader.java @@ -2,7 +2,7 @@ * FileLoader * SPDX-FileCopyrightText: 2010 Michael Peter Christen * SPDX-License-Identifier: GPL-2.0-or-later - * First released 25.5.2010 at http://yacy.net + * First released 25.5.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/crawler/retrieval/RSSLoader.java b/source/net/yacy/crawler/retrieval/RSSLoader.java index 6d1ebd8e96..389913c527 100644 --- a/source/net/yacy/crawler/retrieval/RSSLoader.java +++ b/source/net/yacy/crawler/retrieval/RSSLoader.java @@ -3,7 +3,7 @@ * SPDX-FileCopyrightText: 2010 Michael Peter Christen * SPDX-License-Identifier: GPL-2.0-or-later * Frankfurt a. M., Germany - * First released 27.8.2010 at http://yacy.net + * First released 27.8.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/document/AbstractParser.java b/source/net/yacy/document/AbstractParser.java index f8132be1e7..6a8f9eca81 100644 --- a/source/net/yacy/document/AbstractParser.java +++ b/source/net/yacy/document/AbstractParser.java @@ -1,7 +1,7 @@ /** * Parser * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 29.6.2010 at http://yacy.net + * First released 29.6.2010 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/document/Condenser.java b/source/net/yacy/document/Condenser.java index ef3b2f748b..3c4f73e368 100644 --- a/source/net/yacy/document/Condenser.java +++ b/source/net/yacy/document/Condenser.java @@ -1,7 +1,7 @@ /** * Condenser.java * Copyright 2004 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 09.01.2004 at http://yacy.net + * First released 09.01.2004 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/document/DateDetection.java b/source/net/yacy/document/DateDetection.java index 0b08edd1d6..8bf24e56e9 100644 --- a/source/net/yacy/document/DateDetection.java +++ b/source/net/yacy/document/DateDetection.java @@ -1,7 +1,7 @@ /** * DateDetection * Copyright 2014 by Michael Peter Christen - * First released 12.12.2014 at http://yacy.net + * First released 12.12.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/document/ImageParser.java b/source/net/yacy/document/ImageParser.java index b66518a379..1a1db97f84 100644 --- a/source/net/yacy/document/ImageParser.java +++ b/source/net/yacy/document/ImageParser.java @@ -1,7 +1,7 @@ /** * ImageParser * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 29.6.2010 at http://yacy.net + * First released 29.6.2010 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/document/LargeNumberCache.java b/source/net/yacy/document/LargeNumberCache.java index 9c908b2370..c384245504 100644 --- a/source/net/yacy/document/LargeNumberCache.java +++ b/source/net/yacy/document/LargeNumberCache.java @@ -1,7 +1,7 @@ /** * LargeNumberCache.java * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 09.10.2010 at http://yacy.net + * First released 09.10.2010 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/document/Parser.java b/source/net/yacy/document/Parser.java index f2940a02cd..88904f2909 100644 --- a/source/net/yacy/document/Parser.java +++ b/source/net/yacy/document/Parser.java @@ -1,7 +1,7 @@ /** * Parser.java * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 29.6.2010 at http://yacy.net + * First released 29.6.2010 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/document/Phrase.java b/source/net/yacy/document/Phrase.java index 36ee22e3c9..6b4194fc8e 100644 --- a/source/net/yacy/document/Phrase.java +++ b/source/net/yacy/document/Phrase.java @@ -1,7 +1,7 @@ /** * Phrase.java * Copyright 2008 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 26.03.2008 at http://yacy.net + * First released 26.03.2008 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/document/SnippetExtractor.java b/source/net/yacy/document/SnippetExtractor.java index 94a8ca5c1c..5daf72f967 100644 --- a/source/net/yacy/document/SnippetExtractor.java +++ b/source/net/yacy/document/SnippetExtractor.java @@ -1,7 +1,7 @@ /** * SnippetExtractor * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 22.10.2010 at http://yacy.net + * First released 22.10.2010 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/document/TextParser.java b/source/net/yacy/document/TextParser.java index ae1a001a9b..7584ad6dc4 100644 --- a/source/net/yacy/document/TextParser.java +++ b/source/net/yacy/document/TextParser.java @@ -1,7 +1,7 @@ /** * TextParser.java * Copyright 2009 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 09.07.2009 at http://yacy.net + * First released 09.07.2009 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/document/Tokenizer.java b/source/net/yacy/document/Tokenizer.java index 8ed219bf4a..45ac56940b 100644 --- a/source/net/yacy/document/Tokenizer.java +++ b/source/net/yacy/document/Tokenizer.java @@ -1,7 +1,7 @@ /** * Annotation.java * Copyright 2004 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 09.01.2004 at http://yacy.net + * First released 09.01.2004 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General private diff --git a/source/net/yacy/document/VocabularyScraper.java b/source/net/yacy/document/VocabularyScraper.java index 5328166c7b..4c8124a919 100644 --- a/source/net/yacy/document/VocabularyScraper.java +++ b/source/net/yacy/document/VocabularyScraper.java @@ -1,7 +1,7 @@ /** * VocabularyScraper * Copyright 2015 by Michael Peter Christen - * First released 30.01.2015 at http://yacy.net + * First released 30.01.2015 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/document/importer/Importer.java b/source/net/yacy/document/importer/Importer.java index 7a83d834d0..ba0a2fbfaa 100644 --- a/source/net/yacy/document/importer/Importer.java +++ b/source/net/yacy/document/importer/Importer.java @@ -1,7 +1,7 @@ /** * Importer * Copyright 2009 by Michael Peter Christen - * First released 29.04.2010 at http://yacy.net + * First released 29.04.2010 at https://yacy.net * * This is a part of YaCy, a peer-to-peer based web search engine * diff --git a/source/net/yacy/document/importer/MediawikiImporter.java b/source/net/yacy/document/importer/MediawikiImporter.java index b3d75776d4..bf2260b976 100644 --- a/source/net/yacy/document/importer/MediawikiImporter.java +++ b/source/net/yacy/document/importer/MediawikiImporter.java @@ -1,7 +1,7 @@ /** * MediawikiImporter * Copyright 2008 by Michael Peter Christen - * First released 20.11.2008 at http://yacy.net + * First released 20.11.2008 at https://yacy.net * * This is a part of YaCy, a peer-to-peer based web search engine * diff --git a/source/net/yacy/document/importer/OAIListFriendsLoader.java b/source/net/yacy/document/importer/OAIListFriendsLoader.java index 651513e9e3..8d9c6652bb 100644 --- a/source/net/yacy/document/importer/OAIListFriendsLoader.java +++ b/source/net/yacy/document/importer/OAIListFriendsLoader.java @@ -1,7 +1,7 @@ /** * OAIListFriendsLoader * Copyright 2010 by Michael Peter Christen - * First released 29.04.2010 at http://yacy.net + * First released 29.04.2010 at https://yacy.net * * This is a part of YaCy, a peer-to-peer based web search engine * diff --git a/source/net/yacy/document/importer/OAIPMHImporter.java b/source/net/yacy/document/importer/OAIPMHImporter.java index 5f4a8c3ca0..eae37b0aed 100644 --- a/source/net/yacy/document/importer/OAIPMHImporter.java +++ b/source/net/yacy/document/importer/OAIPMHImporter.java @@ -1,7 +1,7 @@ /** * OAIPMHImporter * Copyright 2009 by Michael Peter Christen - * First released 30.09.2009 at http://yacy.net + * First released 30.09.2009 at https://yacy.net * * This is a part of YaCy, a peer-to-peer based web search engine * diff --git a/source/net/yacy/document/importer/OAIPMHLoader.java b/source/net/yacy/document/importer/OAIPMHLoader.java index 6e4e77d4b5..5afd06d461 100644 --- a/source/net/yacy/document/importer/OAIPMHLoader.java +++ b/source/net/yacy/document/importer/OAIPMHLoader.java @@ -1,7 +1,7 @@ /** * OAIPMHLoader * Copyright 2009 by Michael Peter Christen - * First released 30.09.2009 at http://yacy.net + * First released 30.09.2009 at https://yacy.net * * This is a part of YaCy, a peer-to-peer based web search engine * diff --git a/source/net/yacy/document/importer/ResumptionToken.java b/source/net/yacy/document/importer/ResumptionToken.java index f0c21e6edf..ed69cd2b47 100644 --- a/source/net/yacy/document/importer/ResumptionToken.java +++ b/source/net/yacy/document/importer/ResumptionToken.java @@ -1,7 +1,7 @@ /** * ResumptionToken * Copyright 2009 by Michael Peter Christen - * First released 31.10.2009 at http://yacy.net + * First released 31.10.2009 at https://yacy.net * * This is a part of YaCy, a peer-to-peer based web search engine * diff --git a/source/net/yacy/document/parser/apkParser.java b/source/net/yacy/document/parser/apkParser.java index d14b877683..5f2e507e58 100644 --- a/source/net/yacy/document/parser/apkParser.java +++ b/source/net/yacy/document/parser/apkParser.java @@ -1,7 +1,7 @@ /** * apkParser * Copyright 2014 by Michael Peter Christen - * First released 09.06.2014 at http://yacy.net + * First released 09.06.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/document/parser/audioTagParser.java b/source/net/yacy/document/parser/audioTagParser.java index 925c7fe3e2..c4e18adb13 100644 --- a/source/net/yacy/document/parser/audioTagParser.java +++ b/source/net/yacy/document/parser/audioTagParser.java @@ -1,7 +1,7 @@ /** * mp3Parser * Copyright 2012 by Stefan Foerster, Norderstedt, Germany - * First released 01.10.2012 at http://yacy.net + * First released 01.10.2012 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/document/parser/csvParser.java b/source/net/yacy/document/parser/csvParser.java index cfee1758f9..cf23bcfab3 100644 --- a/source/net/yacy/document/parser/csvParser.java +++ b/source/net/yacy/document/parser/csvParser.java @@ -1,7 +1,7 @@ /** * tarParser * Copyright 2009 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 02.10.2009 at http://yacy.net + * First released 02.10.2009 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/document/parser/genericParser.java b/source/net/yacy/document/parser/genericParser.java index 44d30413ac..4fe41ea537 100644 --- a/source/net/yacy/document/parser/genericParser.java +++ b/source/net/yacy/document/parser/genericParser.java @@ -1,7 +1,7 @@ /** * genericParser * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 30.11.2010 at http://yacy.net + * First released 30.11.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/document/parser/html/EmbedEntry.java b/source/net/yacy/document/parser/html/EmbedEntry.java index 36134a2738..4d01bcf117 100644 --- a/source/net/yacy/document/parser/html/EmbedEntry.java +++ b/source/net/yacy/document/parser/html/EmbedEntry.java @@ -1,7 +1,7 @@ /** * EmbedEntry * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 27.04.2012 at http://yacy.net + * First released 27.04.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/document/parser/html/Evaluation.java b/source/net/yacy/document/parser/html/Evaluation.java index 7d2ba74c5a..b910fe6b78 100644 --- a/source/net/yacy/document/parser/html/Evaluation.java +++ b/source/net/yacy/document/parser/html/Evaluation.java @@ -1,7 +1,7 @@ /** * Evaluation * Copyright 2011 by Michael Peter Christen - * First released 28.04.2011 at http://yacy.net + * First released 28.04.2011 at https://yacy.net * * $LastChangedDate: 2011-03-08 02:51:51 +0100 (Di, 08 Mrz 2011) $ * $LastChangedRevision: 7567 $ diff --git a/source/net/yacy/document/parser/htmlParser.java b/source/net/yacy/document/parser/htmlParser.java index 9609574905..78dce898f5 100644 --- a/source/net/yacy/document/parser/htmlParser.java +++ b/source/net/yacy/document/parser/htmlParser.java @@ -1,7 +1,7 @@ /** * htmlParser.java * Copyright 2009 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 09.07.2009 at http://yacy.net + * First released 09.07.2009 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/document/parser/images/svgParser.java b/source/net/yacy/document/parser/images/svgParser.java index 2045e5afb5..3c44d6e90f 100644 --- a/source/net/yacy/document/parser/images/svgParser.java +++ b/source/net/yacy/document/parser/images/svgParser.java @@ -1,7 +1,7 @@ /** * svgParser.java * Copyright 2015 by Burkhard Buelte - * First released 26.09.2015 at http://yacy.net + * First released 26.09.2015 at https://yacy.net * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free diff --git a/source/net/yacy/document/parser/linkScraperParser.java b/source/net/yacy/document/parser/linkScraperParser.java index dabe3f6312..a21e2ea1eb 100644 --- a/source/net/yacy/document/parser/linkScraperParser.java +++ b/source/net/yacy/document/parser/linkScraperParser.java @@ -1,7 +1,7 @@ /** * linkScraperParser * Copyright 2014 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 07.07.2014 at http://yacy.net + * First released 07.07.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/document/parser/mmParser.java b/source/net/yacy/document/parser/mmParser.java index aef0f8f8a0..849969e17c 100644 --- a/source/net/yacy/document/parser/mmParser.java +++ b/source/net/yacy/document/parser/mmParser.java @@ -1,7 +1,7 @@ /** * mmParser * Copyright 2010 by Marc Nause, marc.nause@gmx.de, Braunschweig, Germany - * First released 27.12.2010 at http://yacy.net + * First released 27.12.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/document/parser/rdfParser.java b/source/net/yacy/document/parser/rdfParser.java index c2302f7c2b..93237c8090 100644 --- a/source/net/yacy/document/parser/rdfParser.java +++ b/source/net/yacy/document/parser/rdfParser.java @@ -1,7 +1,7 @@ /** * rssParser.java * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 20.08.2010 at http://yacy.net + * First released 20.08.2010 at https://yacy.net * * $LastChangedDate: 2011-04-21 15:58:49 +0200 (Do, 21 Apr 2011) $ * $LastChangedRevision: 7672 $ diff --git a/source/net/yacy/document/parser/rssParser.java b/source/net/yacy/document/parser/rssParser.java index 2da3e73da1..930bf9e3d7 100644 --- a/source/net/yacy/document/parser/rssParser.java +++ b/source/net/yacy/document/parser/rssParser.java @@ -1,7 +1,7 @@ /** * rssParser.java * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 20.08.2010 at http://yacy.net + * First released 20.08.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/document/parser/sidAudioParser.java b/source/net/yacy/document/parser/sidAudioParser.java index eb64dfaa03..0cc357ddbc 100644 --- a/source/net/yacy/document/parser/sidAudioParser.java +++ b/source/net/yacy/document/parser/sidAudioParser.java @@ -1,7 +1,7 @@ /** * sidAudioParser * Copyright 2010 by Marc Nause, marc.nause@gmx.de, Braunschweig, Germany - * First released 28.12.2010 at http://yacy.net + * First released 28.12.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/document/parser/sitemapParser.java b/source/net/yacy/document/parser/sitemapParser.java index bfc49ac903..ed6f6b38a1 100644 --- a/source/net/yacy/document/parser/sitemapParser.java +++ b/source/net/yacy/document/parser/sitemapParser.java @@ -1,7 +1,7 @@ /** * sitemapParser.java * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 08.09.2010 at http://yacy.net + * First released 08.09.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/document/parser/tarParser.java b/source/net/yacy/document/parser/tarParser.java index f0122c3cb2..d8840c639f 100644 --- a/source/net/yacy/document/parser/tarParser.java +++ b/source/net/yacy/document/parser/tarParser.java @@ -1,7 +1,7 @@ /** * tarParser * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 29.6.2010 at http://yacy.net + * First released 29.6.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/document/parser/torrentParser.java b/source/net/yacy/document/parser/torrentParser.java index 72f82bd0af..6fbc27791d 100644 --- a/source/net/yacy/document/parser/torrentParser.java +++ b/source/net/yacy/document/parser/torrentParser.java @@ -1,7 +1,7 @@ /** * torrentParser.java * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 03.01.2010 at http://yacy.net + * First released 03.01.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/document/parser/zipParser.java b/source/net/yacy/document/parser/zipParser.java index c994f096f5..731b28d2e0 100644 --- a/source/net/yacy/document/parser/zipParser.java +++ b/source/net/yacy/document/parser/zipParser.java @@ -1,7 +1,7 @@ /** * zipParser * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 29.6.2010 at http://yacy.net + * First released 29.6.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/gui/Audio.java b/source/net/yacy/gui/Audio.java index a0d90c9024..7ee6bbd8ee 100644 --- a/source/net/yacy/gui/Audio.java +++ b/source/net/yacy/gui/Audio.java @@ -1,7 +1,7 @@ /** * Audio * Copyright 2014 by Michael Peter Christen - * First released 07.10.2014 at http://yacy.net + * First released 07.10.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/gui/InfoPage.java b/source/net/yacy/gui/InfoPage.java index fcfc818a3b..4de169c9cd 100644 --- a/source/net/yacy/gui/InfoPage.java +++ b/source/net/yacy/gui/InfoPage.java @@ -1,7 +1,7 @@ /** * InfoPage * Copyright 2010 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * First released 05.08.2010 at http://yacy.net + * First released 05.08.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/gui/YaCyApp.java b/source/net/yacy/gui/YaCyApp.java index 76f2b39615..591e4c169c 100644 --- a/source/net/yacy/gui/YaCyApp.java +++ b/source/net/yacy/gui/YaCyApp.java @@ -1,7 +1,7 @@ /** * YaCyApp * Copyright 2010 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * First released 05.08.2010 at http://yacy.net + * First released 05.08.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/gui/framework/Application.java b/source/net/yacy/gui/framework/Application.java index 2c0ad13357..9bb55bccf4 100644 --- a/source/net/yacy/gui/framework/Application.java +++ b/source/net/yacy/gui/framework/Application.java @@ -1,7 +1,7 @@ /** * Application * Copyright 2010 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * First released 05.08.2010 at http://yacy.net + * First released 05.08.2010 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/gui/framework/Browser.java b/source/net/yacy/gui/framework/Browser.java index 741a300470..80d566127e 100644 --- a/source/net/yacy/gui/framework/Browser.java +++ b/source/net/yacy/gui/framework/Browser.java @@ -1,7 +1,7 @@ /** * Browser * Copyright 2010 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * First released 05.08.2010 at http://yacy.net + * First released 05.08.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/gui/framework/Layout.java b/source/net/yacy/gui/framework/Layout.java index f0fb3fe7f7..ea1393e6f2 100644 --- a/source/net/yacy/gui/framework/Layout.java +++ b/source/net/yacy/gui/framework/Layout.java @@ -1,7 +1,7 @@ /** * Layout * Copyright 2010 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * First released 05.08.2010 at http://yacy.net + * First released 05.08.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/gui/framework/Operation.java b/source/net/yacy/gui/framework/Operation.java index 274e86266d..2f46482e54 100644 --- a/source/net/yacy/gui/framework/Operation.java +++ b/source/net/yacy/gui/framework/Operation.java @@ -1,7 +1,7 @@ /** * Operation * Copyright 2010 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * First released 05.08.2010 at http://yacy.net + * First released 05.08.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/gui/framework/Switchboard.java b/source/net/yacy/gui/framework/Switchboard.java index a5720454df..b2d501a635 100644 --- a/source/net/yacy/gui/framework/Switchboard.java +++ b/source/net/yacy/gui/framework/Switchboard.java @@ -1,7 +1,7 @@ /** * Switchboard * Copyright 2010 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * First released 05.08.2010 at http://yacy.net + * First released 05.08.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/htroot/AccessGrid_p.java b/source/net/yacy/htroot/AccessGrid_p.java index 3174084b95..299d19a28f 100644 --- a/source/net/yacy/htroot/AccessGrid_p.java +++ b/source/net/yacy/htroot/AccessGrid_p.java @@ -1,7 +1,7 @@ /** * AccessGrid_p * Copyright 2011 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * First released 18.02.2010 at http://yacy.net + * First released 18.02.2010 at https://yacy.net * * $LastChangedDate: 2010-06-16 17:11:21 +0200 (Mi, 16 Jun 2010) $ * $LastChangedRevision: 6922 $ diff --git a/source/net/yacy/htroot/AccessPicture_p.java b/source/net/yacy/htroot/AccessPicture_p.java index 782815be43..b8be21567d 100644 --- a/source/net/yacy/htroot/AccessPicture_p.java +++ b/source/net/yacy/htroot/AccessPicture_p.java @@ -1,7 +1,7 @@ /** * AccessPicture_p * Copyright 2011 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * First released 18.02.2010 at http://yacy.net + * First released 18.02.2010 at https://yacy.net * * $LastChangedDate: 2010-06-16 17:11:21 +0200 (Mi, 16 Jun 2010) $ * $LastChangedRevision: 6922 $ diff --git a/source/net/yacy/htroot/ContentAnalysis_p.java b/source/net/yacy/htroot/ContentAnalysis_p.java index a6972cd37e..a30c57ed76 100644 --- a/source/net/yacy/htroot/ContentAnalysis_p.java +++ b/source/net/yacy/htroot/ContentAnalysis_p.java @@ -1,7 +1,7 @@ /** * ContentAnalysis_p * Copyright 2013 by Michael Peter Christen - * First released 12.03.2013 at http://yacy.net + * First released 12.03.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/CrawlCheck_p.java b/source/net/yacy/htroot/CrawlCheck_p.java index 8a55dd4660..d32c6f4429 100644 --- a/source/net/yacy/htroot/CrawlCheck_p.java +++ b/source/net/yacy/htroot/CrawlCheck_p.java @@ -1,7 +1,7 @@ /** * CrawlCheck_p * Copyright 2012 by Michael Peter Christen - * First released 10.10.2011 at http://yacy.net + * First released 10.10.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General private diff --git a/source/net/yacy/htroot/CrawlStartScanner_p.java b/source/net/yacy/htroot/CrawlStartScanner_p.java index d76cf08b37..e414ede9fa 100644 --- a/source/net/yacy/htroot/CrawlStartScanner_p.java +++ b/source/net/yacy/htroot/CrawlStartScanner_p.java @@ -1,7 +1,7 @@ /** * CrawlStartScanner_p * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 12.12.2010 at http://yacy.net + * First released 12.12.2010 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/IndexBrowser_p.java b/source/net/yacy/htroot/IndexBrowser_p.java index 08a7d950ee..d15e51e8d4 100644 --- a/source/net/yacy/htroot/IndexBrowser_p.java +++ b/source/net/yacy/htroot/IndexBrowser_p.java @@ -1,7 +1,7 @@ /** * IndexBrowser * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 27.09.2012 at http://yacy.net + * First released 27.09.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/IndexDeletion_p.java b/source/net/yacy/htroot/IndexDeletion_p.java index af85973436..e97bb9efbd 100644 --- a/source/net/yacy/htroot/IndexDeletion_p.java +++ b/source/net/yacy/htroot/IndexDeletion_p.java @@ -1,7 +1,7 @@ /** * IndexDeletion_p * Copyright 2013 by Michael Peter Christen - * First released 29.04.2013 at http://yacy.net + * First released 29.04.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/IndexFederated_p.java b/source/net/yacy/htroot/IndexFederated_p.java index ec13fd4d59..bbaba2b3a5 100644 --- a/source/net/yacy/htroot/IndexFederated_p.java +++ b/source/net/yacy/htroot/IndexFederated_p.java @@ -1,7 +1,7 @@ /** * IndexFederated_p * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 25.05.2011 at http://yacy.net + * First released 25.05.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/IndexReIndexMonitor_p.java b/source/net/yacy/htroot/IndexReIndexMonitor_p.java index 78e40aac1e..ca31064b9a 100644 --- a/source/net/yacy/htroot/IndexReIndexMonitor_p.java +++ b/source/net/yacy/htroot/IndexReIndexMonitor_p.java @@ -1,7 +1,7 @@ /** * IndexReIndexMonitor_p Copyright 2013 by Michael Peter Christen First released - * 29.04.2013 at http://yacy.net + * 29.04.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free diff --git a/source/net/yacy/htroot/IndexSchema_p.java b/source/net/yacy/htroot/IndexSchema_p.java index ae00ede547..93ed08cce2 100644 --- a/source/net/yacy/htroot/IndexSchema_p.java +++ b/source/net/yacy/htroot/IndexSchema_p.java @@ -1,7 +1,7 @@ /** * IndexSchemaFulltext_p * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 13.02.2013 at http://yacy.net + * First released 13.02.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/Load_RSS_p.java b/source/net/yacy/htroot/Load_RSS_p.java index 05975966c3..565b26dc30 100644 --- a/source/net/yacy/htroot/Load_RSS_p.java +++ b/source/net/yacy/htroot/Load_RSS_p.java @@ -1,7 +1,7 @@ /** * Load_RSS_p * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 20.08.2010 at http://yacy.net + * First released 20.08.2010 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/NetworkHistory.java b/source/net/yacy/htroot/NetworkHistory.java index 043ef3b7fe..83b9817771 100644 --- a/source/net/yacy/htroot/NetworkHistory.java +++ b/source/net/yacy/htroot/NetworkHistory.java @@ -1,7 +1,7 @@ /** * NetworkHistory * Copyright 2014 by Michael Peter Christen - * First released 10.10.2014 at http://yacy.net + * First released 10.10.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/RankingSolr_p.java b/source/net/yacy/htroot/RankingSolr_p.java index f94480e115..9b8a4187bf 100644 --- a/source/net/yacy/htroot/RankingSolr_p.java +++ b/source/net/yacy/htroot/RankingSolr_p.java @@ -1,7 +1,7 @@ /** * RankingSolr_p * Copyright 2012 by Michael Peter Christen - * First released 30.11.2012 at http://yacy.net + * First released 30.11.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/RegexTest.java b/source/net/yacy/htroot/RegexTest.java index c7a5633373..2b40d46ada 100644 --- a/source/net/yacy/htroot/RegexTest.java +++ b/source/net/yacy/htroot/RegexTest.java @@ -1,7 +1,7 @@ /** * RegexTest * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 14.09.2011 at http://yacy.net + * First released 14.09.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/ServerScannerList.java b/source/net/yacy/htroot/ServerScannerList.java index df1defe366..e8182ec909 100644 --- a/source/net/yacy/htroot/ServerScannerList.java +++ b/source/net/yacy/htroot/ServerScannerList.java @@ -1,7 +1,7 @@ /** * CrawlStartScanner_p * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 12.08.2011 at http://yacy.net + * First released 12.08.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/Trails.java b/source/net/yacy/htroot/Trails.java index 77af2d6029..abb919adb6 100644 --- a/source/net/yacy/htroot/Trails.java +++ b/source/net/yacy/htroot/Trails.java @@ -1,7 +1,7 @@ /** * Trails * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 14.09.2011 at http://yacy.net + * First released 14.09.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/Translator_p.java b/source/net/yacy/htroot/Translator_p.java index 8a7906a4e6..872dddd155 100644 --- a/source/net/yacy/htroot/Translator_p.java +++ b/source/net/yacy/htroot/Translator_p.java @@ -2,7 +2,7 @@ /** * Translator_p * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 14.09.2011 at http://yacy.net + * First released 14.09.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/Vocabulary_p.java b/source/net/yacy/htroot/Vocabulary_p.java index 614964eb07..ec47297984 100644 --- a/source/net/yacy/htroot/Vocabulary_p.java +++ b/source/net/yacy/htroot/Vocabulary_p.java @@ -1,7 +1,7 @@ /** * Vocabulary_p * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 07.05.2012 at http://yacy.net + * First released 07.05.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/api/citation.java b/source/net/yacy/htroot/api/citation.java index 355740f233..74ab36767a 100644 --- a/source/net/yacy/htroot/api/citation.java +++ b/source/net/yacy/htroot/api/citation.java @@ -1,7 +1,7 @@ /** * citation * Copyright 2013 by Michael Peter Christen - * First released 12.6.2013 at http://yacy.net + * First released 12.6.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/api/push_p.java b/source/net/yacy/htroot/api/push_p.java index 657a2e15fe..02ff14931b 100644 --- a/source/net/yacy/htroot/api/push_p.java +++ b/source/net/yacy/htroot/api/push_p.java @@ -1,7 +1,7 @@ /** * push_p * Copyright 2014 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 12.06.2014 at http://yacy.net + * First released 12.06.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/api/schema.java b/source/net/yacy/htroot/api/schema.java index 875896a432..71564eae6e 100644 --- a/source/net/yacy/htroot/api/schema.java +++ b/source/net/yacy/htroot/api/schema.java @@ -1,7 +1,7 @@ /** * schema_p * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 13.01.2012 at http://yacy.net + * First released 13.01.2012 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/htroot/api/share.java b/source/net/yacy/htroot/api/share.java index c2ac5cfb59..74da096dd2 100644 --- a/source/net/yacy/htroot/api/share.java +++ b/source/net/yacy/htroot/api/share.java @@ -1,7 +1,7 @@ /** * share * Copyright 2016 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 24.02.2016 at http://yacy.net + * First released 24.02.2016 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/api/snapshot.java b/source/net/yacy/htroot/api/snapshot.java index 3475052dfa..9b69e77514 100644 --- a/source/net/yacy/htroot/api/snapshot.java +++ b/source/net/yacy/htroot/api/snapshot.java @@ -1,7 +1,7 @@ /** * snapshot * Copyright 2014 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 02.12.2014 at http://yacy.net + * First released 02.12.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/osm.java b/source/net/yacy/htroot/osm.java index 1671e89578..d827bbeb7e 100644 --- a/source/net/yacy/htroot/osm.java +++ b/source/net/yacy/htroot/osm.java @@ -1,7 +1,7 @@ /** * osm * Copyright 2008 by Michael Peter Christen - * First released 13.02.2011 at http://yacy.net + * First released 13.02.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/htroot/yacy/idx.java b/source/net/yacy/htroot/yacy/idx.java index 44b979e25d..2811256016 100644 --- a/source/net/yacy/htroot/yacy/idx.java +++ b/source/net/yacy/htroot/yacy/idx.java @@ -1,7 +1,7 @@ /** * idx * Copyright 2011 by Michael Peter Christen - * First released 16.05.2011 at http://yacy.net + * First released 16.05.2011 at https://yacy.net * * $LastChangedDate: 2011-03-08 02:51:51 +0100 (Di, 08 Mrz 2011) $ * $LastChangedRevision: 7567 $ diff --git a/source/net/yacy/http/AbstractRemoteHandler.java b/source/net/yacy/http/AbstractRemoteHandler.java index ed1b89c4a5..826162c983 100644 --- a/source/net/yacy/http/AbstractRemoteHandler.java +++ b/source/net/yacy/http/AbstractRemoteHandler.java @@ -1,7 +1,7 @@ // // AbstractRemoteHandler // Copyright 2011 by Florian Richter -// First released 2011 at http://yacy.net +// First released 2011 at https://yacy.net // // $LastChangedDate$ // $LastChangedRevision$ diff --git a/source/net/yacy/http/MonitorHandler.java b/source/net/yacy/http/MonitorHandler.java index 4862f87f1b..d859a5c3d3 100644 --- a/source/net/yacy/http/MonitorHandler.java +++ b/source/net/yacy/http/MonitorHandler.java @@ -1,7 +1,7 @@ /** * MonitorHandler * Copyright 2014 by Sebastian Gaebel - * First released 15.05.2014 at http://yacy.net + * First released 15.05.2014 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/http/ProxyCacheHandler.java b/source/net/yacy/http/ProxyCacheHandler.java index e2cf4abe06..7c85977953 100644 --- a/source/net/yacy/http/ProxyCacheHandler.java +++ b/source/net/yacy/http/ProxyCacheHandler.java @@ -2,7 +2,7 @@ // ProxyCacheHandler // Copyright 2004 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany // Copyright 2011 by Florian Richter -// First released 2011 at http://yacy.net +// First released 2011 at https://yacy.net // // $LastChangedDate$ // $LastChangedRevision$ diff --git a/source/net/yacy/http/ProxyHandler.java b/source/net/yacy/http/ProxyHandler.java index 2892c00399..95f4dca563 100644 --- a/source/net/yacy/http/ProxyHandler.java +++ b/source/net/yacy/http/ProxyHandler.java @@ -2,7 +2,7 @@ // ProxyHandler // Copyright 2004 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany // Copyright 2011 by Florian Richter -// First released 2011 at http://yacy.net +// First released 2011 at https://yacy.net // // $LastChangedDate$ // $LastChangedRevision$ diff --git a/source/net/yacy/http/YaCyErrorHandler.java b/source/net/yacy/http/YaCyErrorHandler.java index 1096002413..c839388556 100644 --- a/source/net/yacy/http/YaCyErrorHandler.java +++ b/source/net/yacy/http/YaCyErrorHandler.java @@ -2,7 +2,7 @@ // YaCyErrorHandler // ---------------- // Copyright 2014 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany -// First released 2014 at http://yacy.net +// First released 2014 at https://yacy.net // // $LastChangedDate$ // $LastChangedRevision$ diff --git a/source/net/yacy/http/YaCyHttpServer.java b/source/net/yacy/http/YaCyHttpServer.java index 0f6beddee6..4018b2c175 100644 --- a/source/net/yacy/http/YaCyHttpServer.java +++ b/source/net/yacy/http/YaCyHttpServer.java @@ -1,7 +1,7 @@ // // YaCyHttpServer // Copyright 2011 by Florian Richter -// First released 13.04.2011 at http://yacy.net +// First released 13.04.2011 at https://yacy.net // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/http/YaCyLegacyCredential.java b/source/net/yacy/http/YaCyLegacyCredential.java index 25e96c9c1c..840c106307 100644 --- a/source/net/yacy/http/YaCyLegacyCredential.java +++ b/source/net/yacy/http/YaCyLegacyCredential.java @@ -1,7 +1,7 @@ // // YaCyLegacyCredentials // Copyright 2011 by Florian Richter -// First released 16.04.2011 at http://yacy.net +// First released 16.04.2011 at https://yacy.net // // $LastChangedDate$ // $LastChangedRevision$ diff --git a/source/net/yacy/http/YaCyLoginService.java b/source/net/yacy/http/YaCyLoginService.java index cbcd1af927..78b1e1ee91 100644 --- a/source/net/yacy/http/YaCyLoginService.java +++ b/source/net/yacy/http/YaCyLoginService.java @@ -1,7 +1,7 @@ // // YaCyLoginService // Copyright 2011 by Florian Richter -// First released 16.04.2011 at http://yacy.net +// First released 16.04.2011 at https://yacy.net // // $LastChangedDate$ // $LastChangedRevision$ diff --git a/source/net/yacy/http/YaCySecurityHandler.java b/source/net/yacy/http/YaCySecurityHandler.java index 4c1e2e6a59..68948a0815 100644 --- a/source/net/yacy/http/YaCySecurityHandler.java +++ b/source/net/yacy/http/YaCySecurityHandler.java @@ -1,7 +1,7 @@ // // YaCySecurityHandler // Copyright 2011 by Florian Richter -// First released 16.04.2011 at http://yacy.net +// First released 16.04.2011 at https://yacy.net // // $LastChangedDate$ // $LastChangedRevision$ diff --git a/source/net/yacy/http/YacyDomainHandler.java b/source/net/yacy/http/YacyDomainHandler.java index 989b999167..e91048d8f7 100644 --- a/source/net/yacy/http/YacyDomainHandler.java +++ b/source/net/yacy/http/YacyDomainHandler.java @@ -2,7 +2,7 @@ // YacyDomainHandler // Copyright 2005 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany // Copyright 2011 by Florian Richter -// First released 13.04.2011 at http://yacy.net +// First released 13.04.2011 at https://yacy.net // // $LastChangedDate$ // $LastChangedRevision$ diff --git a/source/net/yacy/http/servlets/RAGProxyServlet.java b/source/net/yacy/http/servlets/RAGProxyServlet.java index aa5c708ab2..3dba3cd970 100644 --- a/source/net/yacy/http/servlets/RAGProxyServlet.java +++ b/source/net/yacy/http/servlets/RAGProxyServlet.java @@ -1,7 +1,7 @@ /** * RAGProxyServlet * Copyright 2024 by Michael Peter Christen - * First released 17.05.2024 at http://yacy.net + * First released 17.05.2024 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/http/servlets/SolrSelectServlet.java b/source/net/yacy/http/servlets/SolrSelectServlet.java index ea55907071..74c9fe0041 100644 --- a/source/net/yacy/http/servlets/SolrSelectServlet.java +++ b/source/net/yacy/http/servlets/SolrSelectServlet.java @@ -1,7 +1,7 @@ /** * SolrSelectServlet * Copyright 2012 by Michael Peter Christen - * First released 23.08.2012 at http://yacy.net + * First released 23.08.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/http/servlets/SolrServlet.java b/source/net/yacy/http/servlets/SolrServlet.java index 7b2c621a3d..7e501df457 100644 --- a/source/net/yacy/http/servlets/SolrServlet.java +++ b/source/net/yacy/http/servlets/SolrServlet.java @@ -1,7 +1,7 @@ /** * SolrServlet * Copyright 2014 by Michael Peter Christen - * First released 23.01.2014 at http://yacy.net + * First released 23.01.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/http/servlets/YaCyDefaultServlet.java b/source/net/yacy/http/servlets/YaCyDefaultServlet.java index 1fd3d6c261..235a3fd18e 100644 --- a/source/net/yacy/http/servlets/YaCyDefaultServlet.java +++ b/source/net/yacy/http/servlets/YaCyDefaultServlet.java @@ -1,6 +1,6 @@ // YaCyDefaultServlet // Copyright 2013 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany -// First released 2013 at http://yacy.net +// First released 2013 at https://yacy.net // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/http/servlets/YaCyQoSFilter.java b/source/net/yacy/http/servlets/YaCyQoSFilter.java index e935a38615..b6f9746bfa 100644 --- a/source/net/yacy/http/servlets/YaCyQoSFilter.java +++ b/source/net/yacy/http/servlets/YaCyQoSFilter.java @@ -1,7 +1,7 @@ /** * YaCyQoSFilter * Copyright 2015 by Burkhard Buelte - * First released 26.04.2015 at http://yacy.net + * First released 26.04.2015 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/kelondro/blob/BEncodedHeapBag.java b/source/net/yacy/kelondro/blob/BEncodedHeapBag.java index 1c55cb8d32..c152dba7a4 100644 --- a/source/net/yacy/kelondro/blob/BEncodedHeapBag.java +++ b/source/net/yacy/kelondro/blob/BEncodedHeapBag.java @@ -1,7 +1,7 @@ /** * BEncodedHeapBag * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 16.12.2011 at http://yacy.net + * First released 16.12.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/kelondro/blob/BEncodedHeapShard.java b/source/net/yacy/kelondro/blob/BEncodedHeapShard.java index 180c468774..b4546ee2fc 100644 --- a/source/net/yacy/kelondro/blob/BEncodedHeapShard.java +++ b/source/net/yacy/kelondro/blob/BEncodedHeapShard.java @@ -1,7 +1,7 @@ /** * BEncodedHeapShard * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 16.12.2011 at http://yacy.net + * First released 16.12.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7653 $ diff --git a/source/net/yacy/kelondro/blob/MapColumnIndex.java b/source/net/yacy/kelondro/blob/MapColumnIndex.java index f99d1ce05b..be965ae3a1 100644 --- a/source/net/yacy/kelondro/blob/MapColumnIndex.java +++ b/source/net/yacy/kelondro/blob/MapColumnIndex.java @@ -1,7 +1,7 @@ /** * MapColumnIndex * Copyright 2012 by Michael Christen - * First released 01.02.2012 at http://yacy.net + * First released 01.02.2012 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/kelondro/data/citation/CitationReference.java b/source/net/yacy/kelondro/data/citation/CitationReference.java index f9be7b728f..ac5be19742 100644 --- a/source/net/yacy/kelondro/data/citation/CitationReference.java +++ b/source/net/yacy/kelondro/data/citation/CitationReference.java @@ -1,7 +1,7 @@ /** * CitationReferenceRow * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 13.02.2012 at http://yacy.net + * First released 13.02.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/kelondro/data/citation/CitationReferenceFactory.java b/source/net/yacy/kelondro/data/citation/CitationReferenceFactory.java index 12709e2dc2..4d106b6c51 100644 --- a/source/net/yacy/kelondro/data/citation/CitationReferenceFactory.java +++ b/source/net/yacy/kelondro/data/citation/CitationReferenceFactory.java @@ -1,7 +1,7 @@ /** * CitationReferenceFactory * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 13.02.2012 at http://yacy.net + * First released 13.02.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/kelondro/data/meta/URIMetadataNode.java b/source/net/yacy/kelondro/data/meta/URIMetadataNode.java index 7c2bf2d073..28e3d14b70 100644 --- a/source/net/yacy/kelondro/data/meta/URIMetadataNode.java +++ b/source/net/yacy/kelondro/data/meta/URIMetadataNode.java @@ -1,7 +1,7 @@ /** * URIMetadataNode * Copyright 2012 by Michael Peter Christen - * First released 10.8.2012 at http://yacy.net + * First released 10.8.2012 at https://yacy.net * * This file is part of YaCy Content Integration * diff --git a/source/net/yacy/kelondro/index/BufferedObjectIndex.java b/source/net/yacy/kelondro/index/BufferedObjectIndex.java index 186f95bfcc..9a526d5e15 100644 --- a/source/net/yacy/kelondro/index/BufferedObjectIndex.java +++ b/source/net/yacy/kelondro/index/BufferedObjectIndex.java @@ -1,7 +1,7 @@ /** * BufferedObjectIndex * Copyright 2010 by Michael Peter Christen - * First released 18.4.2010 at http://yacy.net + * First released 18.4.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/kelondro/index/Cache.java b/source/net/yacy/kelondro/index/Cache.java index 226d9b459c..bbd634f8b8 100644 --- a/source/net/yacy/kelondro/index/Cache.java +++ b/source/net/yacy/kelondro/index/Cache.java @@ -1,7 +1,7 @@ /** * Cache.java * Copyright 2006 by Michael Peter Christen - * First released 26.10.2006 at http://yacy.net + * First released 26.10.2006 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/kelondro/index/Index.java b/source/net/yacy/kelondro/index/Index.java index 1dab22376b..bc83c9695d 100644 --- a/source/net/yacy/kelondro/index/Index.java +++ b/source/net/yacy/kelondro/index/Index.java @@ -1,7 +1,7 @@ /** * Index * Copyright 2005 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * First released 26.10.2005 at http://yacy.net + * First released 26.10.2005 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/kelondro/index/OnDemandOpenFileIndex.java b/source/net/yacy/kelondro/index/OnDemandOpenFileIndex.java index db6aa5c8b2..2317787f34 100644 --- a/source/net/yacy/kelondro/index/OnDemandOpenFileIndex.java +++ b/source/net/yacy/kelondro/index/OnDemandOpenFileIndex.java @@ -1,7 +1,7 @@ /** * OnDemandOpenFileIndex * Copyright 2014 by Michael Christen - * First released 16.04.2014 at http://yacy.net + * First released 16.04.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/kelondro/index/RAMIndex.java b/source/net/yacy/kelondro/index/RAMIndex.java index 6ac7bde1f9..5548e1e088 100644 --- a/source/net/yacy/kelondro/index/RAMIndex.java +++ b/source/net/yacy/kelondro/index/RAMIndex.java @@ -1,7 +1,7 @@ /** * RAMIndex * Copyright 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * First released 07.01.2008 at http://yacy.net + * First released 07.01.2008 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/kelondro/index/RAMIndexCluster.java b/source/net/yacy/kelondro/index/RAMIndexCluster.java index 8d952af288..993ecfa1f1 100644 --- a/source/net/yacy/kelondro/index/RAMIndexCluster.java +++ b/source/net/yacy/kelondro/index/RAMIndexCluster.java @@ -1,7 +1,7 @@ /** * RAMIndexCluster * Copyright 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * First released 12.03.2009 at http://yacy.net + * First released 12.03.2009 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/kelondro/index/RowHandleMap.java b/source/net/yacy/kelondro/index/RowHandleMap.java index 0d7eaf4131..555bb0d616 100644 --- a/source/net/yacy/kelondro/index/RowHandleMap.java +++ b/source/net/yacy/kelondro/index/RowHandleMap.java @@ -1,7 +1,7 @@ /** * HandleMap * Copyright 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * First released 08.04.2008 at http://yacy.net + * First released 08.04.2008 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/kelondro/index/RowSet.java b/source/net/yacy/kelondro/index/RowSet.java index e286a86a37..aa4fff910e 100644 --- a/source/net/yacy/kelondro/index/RowSet.java +++ b/source/net/yacy/kelondro/index/RowSet.java @@ -1,7 +1,7 @@ /** * RowSet * Copyright 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * First released 20.06.2006 at http://yacy.net + * First released 20.06.2006 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/kelondro/table/IndexTable.java b/source/net/yacy/kelondro/table/IndexTable.java index cb02fd0d7a..208921760d 100644 --- a/source/net/yacy/kelondro/table/IndexTable.java +++ b/source/net/yacy/kelondro/table/IndexTable.java @@ -1,7 +1,7 @@ /** * IndexTable * Copyright 2014 by Michael Peter Christen - * First released 11.11.2014 at http://yacy.net + * First released 11.11.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/peers/EventChannel.java b/source/net/yacy/peers/EventChannel.java index 25ac25f9bb..513420448b 100644 --- a/source/net/yacy/peers/EventChannel.java +++ b/source/net/yacy/peers/EventChannel.java @@ -1,7 +1,7 @@ /** * yacyChannel * Copyright 2010 by Michael Peter Christen - * First released 29.6.2010 at http://yacy.net + * First released 29.6.2010 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/peers/graphics/EncodedImage.java b/source/net/yacy/peers/graphics/EncodedImage.java index c54e963792..f6de55ebd5 100644 --- a/source/net/yacy/peers/graphics/EncodedImage.java +++ b/source/net/yacy/peers/graphics/EncodedImage.java @@ -1,7 +1,7 @@ /** * EncodedImage * Copyright 2010 by Michael Christen - * First released 28.07.2010 at http://yacy.net + * First released 28.07.2010 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/peers/graphics/OSMTile.java b/source/net/yacy/peers/graphics/OSMTile.java index 019e9da0aa..8dc903764f 100644 --- a/source/net/yacy/peers/graphics/OSMTile.java +++ b/source/net/yacy/peers/graphics/OSMTile.java @@ -1,7 +1,7 @@ /** * OSMTile * Copyright 2008 by Michael Peter Christen - * First released 12.02.2008 at http://yacy.net + * First released 12.02.2008 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/search/AutoSearch.java b/source/net/yacy/search/AutoSearch.java index 980de4ce5d..524a2e8911 100644 --- a/source/net/yacy/search/AutoSearch.java +++ b/source/net/yacy/search/AutoSearch.java @@ -1,7 +1,7 @@ /** * AutoSearch.java * Copyright 2015 by Burkhard Buelte - * First released 09.01.2015 at http://yacy.net + * First released 09.01.2015 at https://yacy.net * * This is a part of YaCy, a peer-to-peer based web search engine * diff --git a/source/net/yacy/search/IndexingQueueEntry.java b/source/net/yacy/search/IndexingQueueEntry.java index 818356a599..0c26cd129a 100644 --- a/source/net/yacy/search/IndexingQueueEntry.java +++ b/source/net/yacy/search/IndexingQueueEntry.java @@ -1,7 +1,7 @@ /** * IndexingQueueEntry * Copyright 2012 by Michael Peter Christen - * First released 24.07.2012 at http://yacy.net + * First released 24.07.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/search/Shutdown.java b/source/net/yacy/search/Shutdown.java index 68705c3a48..c303e4e07a 100644 --- a/source/net/yacy/search/Shutdown.java +++ b/source/net/yacy/search/Shutdown.java @@ -1,7 +1,7 @@ /** * Shutdown * Copyright 2012 by Michael Peter Christen - * First released 24.07.2012 at http://yacy.net + * First released 24.07.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 0c39b51cf7..20778899c2 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -103,6 +103,7 @@ import com.cybozu.labs.langdetect.LangDetectException; import com.google.common.io.Files; +import net.yacy.yacy; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.WordCache; import net.yacy.cora.document.analysis.Classification; @@ -260,10 +261,9 @@ public final class Switchboard extends serverSwitch { public static WikiParser wikiParser = null; // storage management - public File htCachePath; public final File dictionariesPath, classificationPath; public File listsPath; - public File htDocsPath; + public File htDocsPath, htRootPath, htCachePath; public File workPath; public File releasePath; public File networkRoot; @@ -373,11 +373,12 @@ public boolean jobImpl() throws Exception { this.log.config("Index Primary Path: " + indexPath.toString()); final File archivePath = this.getDataPath(SwitchboardConstants.INDEX_ARCHIVE_PATH, SwitchboardConstants.INDEX_ARCHIVE_DEFAULT); this.log.config("Index Archive Path: " + archivePath.toString()); - this.listsPath = - this.getDataPath(SwitchboardConstants.LISTS_PATH, SwitchboardConstants.LISTS_PATH_DEFAULT); + this.listsPath = this.getDataPath(SwitchboardConstants.LISTS_PATH, SwitchboardConstants.LISTS_PATH_DEFAULT); this.log.config("Lists Path: " + this.listsPath.toString()); - this.htDocsPath = - this.getDataPath(SwitchboardConstants.HTDOCS_PATH, SwitchboardConstants.HTDOCS_PATH_DEFAULT); + this.htRootPath = this.getDataPath(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT); // path to the servlets + this.log.config("HTROOT Path: " + this.htRootPath.toString()); + this.htDocsPath = this.getDataPath(SwitchboardConstants.HTDOCS_PATH, SwitchboardConstants.HTDOCS_PATH_DEFAULT); // a mirror to htroot + yacy.mkdirIfNeseccary(this.htDocsPath); this.log.config("HTDOCS Path: " + this.htDocsPath.toString()); this.workPath = this.getDataPath(SwitchboardConstants.WORK_PATH, SwitchboardConstants.WORK_PATH_DEFAULT); this.workPath.mkdirs(); @@ -590,658 +591,664 @@ public void run() { final String solrurls = this.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"); final boolean usesolr = this.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED_DEFAULT) & solrurls.length() > 0; - final int solrtimeout = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 60000); - final boolean writeEnabled = this.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, true); - final boolean trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool( - SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED, - SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT); + final int solrtimeout = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 60000); + final boolean writeEnabled = this.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, true); + final boolean trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool( + SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED, + SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT); - if (usesolr && solrurls != null && solrurls.length() > 0) { - try { - final ArrayList instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer); - final String shardMethodName = this.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name()); - final ShardSelection.Method shardMethod = ShardSelection.Method.valueOf(shardMethodName); - this.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled); - } catch (final IOException e ) { - ConcurrentLog.logException(e); - } - } + if (usesolr && solrurls != null && solrurls.length() > 0) { + try { + final ArrayList instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer); + final String shardMethodName = this.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name()); + final ShardSelection.Method shardMethod = ShardSelection.Method.valueOf(shardMethodName); + this.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled); + } catch (final IOException e ) { + ConcurrentLog.logException(e); + } + } - // initialize network database - final File mySeedFile = new File(this.networkRoot, SeedDB.DBFILE_OWN_SEED); - this.peers = - new SeedDB( - this.networkRoot, - "seed.new.heap", - "seed.old.heap", - "seed.pot.heap", - mySeedFile, - redundancy, - partitionExponent, - false, - this.exceed134217727); - final String agent = this.getConfig(SwitchboardConstants.NETWORK_UNIT_AGENT, ""); - if (!agent.isEmpty()) this.peers.setMyName(agent); // this can thus be set using the environment variable yacy.network.unit.agent - - // initialize peer scan - this.localcluster_scan = Collections.newSetFromMap(new ConcurrentHashMap<>()); - if (this.getConfigBool("scan.enabled", false)) { - new OneTimeBusyThread("Switchboard.scanForOtherYaCyInIntranet") { - @Override - public boolean jobImpl() throws Exception { - Switchboard.this.localcluster_scan.addAll(Scanner.scanForOtherYaCyInIntranet()); - return true; - } - }.start(); + // initialize network database + final File mySeedFile = new File(this.networkRoot, SeedDB.DBFILE_OWN_SEED); + this.peers = + new SeedDB( + this.networkRoot, + "seed.new.heap", + "seed.old.heap", + "seed.pot.heap", + mySeedFile, + redundancy, + partitionExponent, + false, + this.exceed134217727); + final String agent = this.getConfig(SwitchboardConstants.NETWORK_UNIT_AGENT, ""); + if (!agent.isEmpty()) this.peers.setMyName(agent); // this can thus be set using the environment variable yacy.network.unit.agent + + // initialize peer scan + this.localcluster_scan = Collections.newSetFromMap(new ConcurrentHashMap<>()); + if (this.getConfigBool("scan.enabled", false)) { + new OneTimeBusyThread("Switchboard.scanForOtherYaCyInIntranet") { + @Override + public boolean jobImpl() throws Exception { + Switchboard.this.localcluster_scan.addAll(Scanner.scanForOtherYaCyInIntranet()); + return true; } + }.start(); + } - // load domainList - try { - this.domainList = null; - if (!this.getConfig("network.unit.domainlist", "").equals("")) { - final Reader r = this.getConfigFileFromWebOrLocally( - this.getConfig("network.unit.domainlist", ""), - this.getAppPath().getAbsolutePath(), - new File(this.networkRoot, "domainlist.txt")); - this.domainList = new FilterEngine(); - final BufferedReader br = new BufferedReader(r); - this.domainList.loadList(br, null); - br.close(); - } - } catch (final FileNotFoundException e ) { - this.log.severe("CONFIG: domainlist not found: " + e.getMessage()); - } catch (final IOException e ) { - this.log.severe("CONFIG: error while retrieving domainlist: " + e.getMessage()); - } + // load domainList + try { + this.domainList = null; + if (!this.getConfig("network.unit.domainlist", "").equals("")) { + final Reader r = this.getConfigFileFromWebOrLocally( + this.getConfig("network.unit.domainlist", ""), + this.getAppPath().getAbsolutePath(), + new File(this.networkRoot, "domainlist.txt")); + this.domainList = new FilterEngine(); + final BufferedReader br = new BufferedReader(r); + this.domainList.loadList(br, null); + br.close(); + } + } catch (final FileNotFoundException e ) { + this.log.severe("CONFIG: domainlist not found: " + e.getMessage()); + } catch (final IOException e ) { + this.log.severe("CONFIG: error while retrieving domainlist: " + e.getMessage()); + } - // create a crawler - this.crawler = new CrawlSwitchboard(this); + // create a crawler + this.crawler = new CrawlSwitchboard(this); - // start yacy core - this.log.config("Starting YaCy Protocol Core"); - this.yc = new Network(this); - new OneTimeBusyThread("Switchboard.loadSeedLists") { + // start yacy core + this.log.config("Starting YaCy Protocol Core"); + this.yc = new Network(this); + new OneTimeBusyThread("Switchboard.loadSeedLists") { - @Override - public boolean jobImpl() throws Exception { - Switchboard.this.loadSeedLists(); - return true; - } - }.start(); - //final long startedSeedListAquisition = System.currentTimeMillis(); - - // init a DHT transmission dispatcher - this.dhtDispatcher = (this.peers.sizeConnected() == 0) ? null : new Dispatcher(this, true, 10000); - - // set up local robots.txt - this.robotstxtConfig = RobotsTxtConfig.init(this); - - // setting timestamp of last proxy access - this.proxyLastAccess = System.currentTimeMillis() - 10000; - this.localSearchLastAccess = System.currentTimeMillis() - 10000; - this.remoteSearchLastAccess = System.currentTimeMillis() - 10000; - this.adminAuthenticationLastAccess = 0; // timestamp last admin authentication (as not autenticated here, stamp with 0) - this.optimizeLastRun = System.currentTimeMillis(); - this.webStructure = new WebStructureGraph(new File(this.queuesRoot, "webStructure.map")); - - // configuring list path - if ( !(this.listsPath.exists()) ) { - this.listsPath.mkdirs(); - } + @Override + public boolean jobImpl() throws Exception { + Switchboard.this.loadSeedLists(); + return true; + } + }.start(); + //final long startedSeedListAquisition = System.currentTimeMillis(); + + // init a DHT transmission dispatcher + this.dhtDispatcher = (this.peers.sizeConnected() == 0) ? null : new Dispatcher(this, true, 10000); + + // set up local robots.txt + this.robotstxtConfig = RobotsTxtConfig.init(this); + + // setting timestamp of last proxy access + this.proxyLastAccess = System.currentTimeMillis() - 10000; + this.localSearchLastAccess = System.currentTimeMillis() - 10000; + this.remoteSearchLastAccess = System.currentTimeMillis() - 10000; + this.adminAuthenticationLastAccess = 0; // timestamp last admin authentication (as not autenticated here, stamp with 0) + this.optimizeLastRun = System.currentTimeMillis(); + this.webStructure = new WebStructureGraph(new File(this.queuesRoot, "webStructure.map")); + + // configuring list path + if ( !(this.listsPath.exists()) ) { + this.listsPath.mkdirs(); + } + + // load coloured lists + if ( blueList == null ) { + // read only once upon first instantiation of this class + final String f = + this.getConfig(SwitchboardConstants.LIST_BLUE, SwitchboardConstants.LIST_BLUE_DEFAULT); + final File plasmaBlueListFile = new File(f); + if ( f != null ) { + blueList = SetTools.loadList(plasmaBlueListFile, NaturalOrder.naturalComparator); + } else { + blueList = new TreeSet<>(); + } + this.log.config("loaded blue-list from file " + + plasmaBlueListFile.getName() + + ", " + + blueList.size() + + " entries, " + + ppRamString(plasmaBlueListFile.length() / 1024)); + } - // load coloured lists - if ( blueList == null ) { - // read only once upon first instantiation of this class - final String f = - this.getConfig(SwitchboardConstants.LIST_BLUE, SwitchboardConstants.LIST_BLUE_DEFAULT); - final File plasmaBlueListFile = new File(f); - if ( f != null ) { - blueList = SetTools.loadList(plasmaBlueListFile, NaturalOrder.naturalComparator); - } else { - blueList = new TreeSet<>(); - } - this.log.config("loaded blue-list from file " - + plasmaBlueListFile.getName() - + ", " - + blueList.size() - + " entries, " - + ppRamString(plasmaBlueListFile.length() / 1024)); - } + // load blacklist + this.log.config("Loading blacklist ..."); + final File blacklistsPath = + this.getDataPath(SwitchboardConstants.LISTS_PATH, SwitchboardConstants.LISTS_PATH_DEFAULT); + urlBlacklist = new Blacklist(blacklistsPath); + ListManager.switchboard = this; + ListManager.listsPath = blacklistsPath; + ListManager.reloadBlacklists(); + + // Set jvm default locale to match UI language ( + String lng = this.getConfig("locale.language", "en"); + if (!"browser".equals(lng) && !"default".equals(lng)) { + Locale.setDefault(new Locale(lng)); + } else { + lng = "en"; // default = English + } - // load blacklist - this.log.config("Loading blacklist ..."); - final File blacklistsPath = - this.getDataPath(SwitchboardConstants.LISTS_PATH, SwitchboardConstants.LISTS_PATH_DEFAULT); - urlBlacklist = new Blacklist(blacklistsPath); - ListManager.switchboard = this; - ListManager.listsPath = blacklistsPath; - ListManager.reloadBlacklists(); - - // Set jvm default locale to match UI language ( - String lng = this.getConfig("locale.language", "en"); - if (!"browser".equals(lng) && !"default".equals(lng)) { - Locale.setDefault(new Locale(lng)); - } else { - lng = "en"; // default = English + // load badwords (to filter the topwords) + if ( badwords == null || badwords.isEmpty() ) { + File badwordsFile = new File(appPath, "DATA/SETTINGS/" + SwitchboardConstants.LIST_BADWORDS_DEFAULT); + if (!badwordsFile.exists()) { + badwordsFile = new File(appPath, "defaults/" + SwitchboardConstants.LIST_BADWORDS_DEFAULT); + } + badwords = SetTools.loadList(badwordsFile, NaturalOrder.naturalComparator); + this.log.config("loaded badwords from file " + + badwordsFile.getName() + + ", " + + badwords.size() + + " entries, " + + ppRamString(badwordsFile.length() / 1024)); + } + + // load stopwords (to filter query and topwords) + if ( stopwords == null || stopwords.isEmpty() ) { + File stopwordsFile = new File(dataPath, "DATA/SETTINGS/" + SwitchboardConstants.LIST_STOPWORDS_DEFAULT); + if (!stopwordsFile.exists()) { + stopwordsFile = new File(appPath, "defaults/"+SwitchboardConstants.LIST_STOPWORDS_DEFAULT); + } + stopwords = SetTools.loadList(stopwordsFile, NaturalOrder.naturalComparator); + // append locale language stopwords using setting of interface language (file yacy.stopwords.xx) + // english is stored as default (needed for locale html file overlay) + File stopwordsFilelocale = new File (dataPath, "DATA/SETTINGS/"+stopwordsFile.getName()+"."+lng); + if (!stopwordsFilelocale.exists()) stopwordsFilelocale = new File (appPath, "defaults/"+stopwordsFile.getName()+"."+lng); + if (stopwordsFilelocale.exists()) { + // load YaCy locale stopword list + stopwords.addAll(SetTools.loadList(stopwordsFilelocale, NaturalOrder.naturalComparator)); + this.log.config("append stopwords from file " + stopwordsFilelocale.getName()); + } else { + // alternatively load/append default solr stopword list + stopwordsFilelocale = new File (appPath, "defaults/solr/lang/stopwords_" + lng + ".txt"); + if (stopwordsFilelocale.exists()) { + stopwords.addAll(SetTools.loadList(stopwordsFilelocale, NaturalOrder.naturalComparator)); + this.log.config("append stopwords from file " + stopwordsFilelocale.getName()); } + } + } - // load badwords (to filter the topwords) - if ( badwords == null || badwords.isEmpty() ) { - File badwordsFile = new File(appPath, "DATA/SETTINGS/" + SwitchboardConstants.LIST_BADWORDS_DEFAULT); - if (!badwordsFile.exists()) { - badwordsFile = new File(appPath, "defaults/" + SwitchboardConstants.LIST_BADWORDS_DEFAULT); - } - badwords = SetTools.loadList(badwordsFile, NaturalOrder.naturalComparator); - this.log.config("loaded badwords from file " - + badwordsFile.getName() - + ", " - + badwords.size() - + " entries, " - + ppRamString(badwordsFile.length() / 1024)); - } + // start a cache manager + this.log.config("Starting HT Cache Manager"); + + // create the cache directory + this.htCachePath = + this.getDataPath(SwitchboardConstants.HTCACHE_PATH, SwitchboardConstants.HTCACHE_PATH_DEFAULT); + this.log.info("HTCACHE Path = " + this.htCachePath.getAbsolutePath()); + final long maxCacheSize = + 1024L * 1024L * Long.parseLong(this.getConfig(SwitchboardConstants.PROXY_CACHE_SIZE, "2")); // this is megabyte + Cache.init(this.htCachePath, this.peers.mySeed().hash, maxCacheSize, + this.getConfigLong(SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT, + SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT_DEFAULT), + this.getConfigInt(SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL, + SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL_DEFAULT)); + final File transactiondir = new File(this.htCachePath, "snapshots"); + Transactions.init(transactiondir, this.getConfigLong(SwitchboardConstants.SNAPSHOTS_WKHTMLTOPDF_TIMEOUT, + SwitchboardConstants.SNAPSHOTS_WKHTMLTOPDF_TIMEOUT_DEFAULT)); + + // create the surrogates directories + this.surrogatesInPath = + this.getDataPath( + SwitchboardConstants.SURROGATES_IN_PATH, + SwitchboardConstants.SURROGATES_IN_PATH_DEFAULT); + this.log.info("surrogates.in Path = " + this.surrogatesInPath.getAbsolutePath()); + this.surrogatesInPath.mkdirs(); + this.surrogatesOutPath = + this.getDataPath( + SwitchboardConstants.SURROGATES_OUT_PATH, + SwitchboardConstants.SURROGATES_OUT_PATH_DEFAULT); + this.log.info("surrogates.out Path = " + this.surrogatesOutPath.getAbsolutePath()); + this.surrogatesOutPath.mkdirs(); + + // copy opensearch heuristic config (if not exist) + final File osdConfig = new File(this.getDataPath(), "DATA/SETTINGS/heuristicopensearch.conf"); + if (!osdConfig.exists()) { + final File osdDefaultConfig = new File(appPath, "defaults/heuristicopensearch.conf"); + this.log.info("heuristic.opensearch list Path = " + osdDefaultConfig.getAbsolutePath()); + try { + Files.copy(osdDefaultConfig, osdConfig); + } catch (final IOException ex) { } + } - // load stopwords (to filter query and topwords) - if ( stopwords == null || stopwords.isEmpty() ) { - File stopwordsFile = new File(dataPath, "DATA/SETTINGS/" + SwitchboardConstants.LIST_STOPWORDS_DEFAULT); - if (!stopwordsFile.exists()) { - stopwordsFile = new File(appPath, "defaults/"+SwitchboardConstants.LIST_STOPWORDS_DEFAULT); - } - stopwords = SetTools.loadList(stopwordsFile, NaturalOrder.naturalComparator); - // append locale language stopwords using setting of interface language (file yacy.stopwords.xx) - // english is stored as default (needed for locale html file overlay) - File stopwordsFilelocale = new File (dataPath, "DATA/SETTINGS/"+stopwordsFile.getName()+"."+lng); - if (!stopwordsFilelocale.exists()) stopwordsFilelocale = new File (appPath, "defaults/"+stopwordsFile.getName()+"."+lng); - if (stopwordsFilelocale.exists()) { - // load YaCy locale stopword list - stopwords.addAll(SetTools.loadList(stopwordsFilelocale, NaturalOrder.naturalComparator)); - this.log.config("append stopwords from file " + stopwordsFilelocale.getName()); - } else { - // alternatively load/append default solr stopword list - stopwordsFilelocale = new File (appPath, "defaults/solr/lang/stopwords_" + lng + ".txt"); - if (stopwordsFilelocale.exists()) { - stopwords.addAll(SetTools.loadList(stopwordsFilelocale, NaturalOrder.naturalComparator)); - this.log.config("append stopwords from file " + stopwordsFilelocale.getName()); - } - } - } + // create the release download directory + this.releasePath = + this.getDataPath(SwitchboardConstants.RELEASE_PATH, SwitchboardConstants.RELEASE_PATH_DEFAULT); + this.releasePath.mkdirs(); + this.log.info("RELEASE Path = " + this.releasePath.getAbsolutePath()); - // start a cache manager - this.log.config("Starting HT Cache Manager"); - - // create the cache directory - this.htCachePath = - this.getDataPath(SwitchboardConstants.HTCACHE_PATH, SwitchboardConstants.HTCACHE_PATH_DEFAULT); - this.log.info("HTCACHE Path = " + this.htCachePath.getAbsolutePath()); - final long maxCacheSize = - 1024L * 1024L * Long.parseLong(this.getConfig(SwitchboardConstants.PROXY_CACHE_SIZE, "2")); // this is megabyte - Cache.init(this.htCachePath, this.peers.mySeed().hash, maxCacheSize, - this.getConfigLong(SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT, - SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT_DEFAULT), - this.getConfigInt(SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL, - SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL_DEFAULT)); - final File transactiondir = new File(this.htCachePath, "snapshots"); - Transactions.init(transactiondir, this.getConfigLong(SwitchboardConstants.SNAPSHOTS_WKHTMLTOPDF_TIMEOUT, - SwitchboardConstants.SNAPSHOTS_WKHTMLTOPDF_TIMEOUT_DEFAULT)); - - // create the surrogates directories - this.surrogatesInPath = - this.getDataPath( - SwitchboardConstants.SURROGATES_IN_PATH, - SwitchboardConstants.SURROGATES_IN_PATH_DEFAULT); - this.log.info("surrogates.in Path = " + this.surrogatesInPath.getAbsolutePath()); - this.surrogatesInPath.mkdirs(); - this.surrogatesOutPath = - this.getDataPath( - SwitchboardConstants.SURROGATES_OUT_PATH, - SwitchboardConstants.SURROGATES_OUT_PATH_DEFAULT); - this.log.info("surrogates.out Path = " + this.surrogatesOutPath.getAbsolutePath()); - this.surrogatesOutPath.mkdirs(); - - // copy opensearch heuristic config (if not exist) - final File osdConfig = new File(this.getDataPath(), "DATA/SETTINGS/heuristicopensearch.conf"); - if (!osdConfig.exists()) { - final File osdDefaultConfig = new File(appPath, "defaults/heuristicopensearch.conf"); - this.log.info("heuristic.opensearch list Path = " + osdDefaultConfig.getAbsolutePath()); - try { - Files.copy(osdDefaultConfig, osdConfig); - } catch (final IOException ex) { } - } + // starting message board + try { + this.initMessages(); + } catch (final IOException e) { + ConcurrentLog.logException(e); + } - // create the release download directory - this.releasePath = - this.getDataPath(SwitchboardConstants.RELEASE_PATH, SwitchboardConstants.RELEASE_PATH_DEFAULT); - this.releasePath.mkdirs(); - this.log.info("RELEASE Path = " + this.releasePath.getAbsolutePath()); + // starting wiki + try { + this.initWiki(); + } catch (final IOException e) { + ConcurrentLog.logException(e); + } - // starting message board - try { - this.initMessages(); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } + //starting blog + try { + this.initBlog(); + } catch (final IOException e) { + ConcurrentLog.logException(e); + } - // starting wiki + // init User DB + this.log.config("Loading User DB"); + final File userDbFile = new File(this.getDataPath(), "DATA/SETTINGS/user.heap"); + try { + this.userDB = new UserDB(userDbFile); + this.log.config("Loaded User DB from file " + + userDbFile.getName() + + ", " + + this.userDB.size() + + " entries" + + ", " + + ppRamString(userDbFile.length() / 1024)); + } catch (final IOException e) { + ConcurrentLog.logException(e); + } + + // init html parser evaluation scheme + File parserPropertiesPath = new File(appPath, "defaults/"); + String[] settingsList = parserPropertiesPath.list(); + for ( final String l : settingsList ) { + if ( l.startsWith("parser.") && l.endsWith(".properties") ) { try { - this.initWiki(); + Evaluation.add(new File(parserPropertiesPath, l)); } catch (final IOException e) { ConcurrentLog.logException(e); } - - //starting blog + } + } + parserPropertiesPath = new File(this.getDataPath(), "DATA/SETTINGS/"); + settingsList = parserPropertiesPath.list(); + for ( final String l : settingsList ) { + if ( l.startsWith("parser.") && l.endsWith(".properties") ) { try { - this.initBlog(); + Evaluation.add(new File(parserPropertiesPath, l)); } catch (final IOException e) { ConcurrentLog.logException(e); } + } + } - // init User DB - this.log.config("Loading User DB"); - final File userDbFile = new File(this.getDataPath(), "DATA/SETTINGS/user.heap"); + // init bookmarks DB: needs more time since this does a DNS lookup for each Bookmark. + // Can be started concurrently + new Thread("Switchboard.initBookmarks") { + @Override + public void run() { try { - this.userDB = new UserDB(userDbFile); - this.log.config("Loaded User DB from file " - + userDbFile.getName() - + ", " - + this.userDB.size() - + " entries" - + ", " - + ppRamString(userDbFile.length() / 1024)); - } catch (final IOException e) { + Switchboard.this.initBookmarks(); + } catch (final IOException e ) { ConcurrentLog.logException(e); } + } + }.start(); - // init html parser evaluation scheme - File parserPropertiesPath = new File(appPath, "defaults/"); - String[] settingsList = parserPropertiesPath.list(); - for ( final String l : settingsList ) { - if ( l.startsWith("parser.") && l.endsWith(".properties") ) { - try { - Evaluation.add(new File(parserPropertiesPath, l)); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } - } - } - parserPropertiesPath = new File(this.getDataPath(), "DATA/SETTINGS/"); - settingsList = parserPropertiesPath.list(); - for ( final String l : settingsList ) { - if ( l.startsWith("parser.") && l.endsWith(".properties") ) { - try { - Evaluation.add(new File(parserPropertiesPath, l)); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } - } - } - - // init bookmarks DB: needs more time since this does a DNS lookup for each Bookmark. - // Can be started concurrently - new Thread("Switchboard.initBookmarks") { - @Override - public void run() { - try { - Switchboard.this.initBookmarks(); - } catch (final IOException e ) { - ConcurrentLog.logException(e); - } - } - }.start(); - - // define a realtime parsable mimetype list - this.log.config("Parser: Initializing Mime Type deny list"); + // define a realtime parsable mimetype list + this.log.config("Parser: Initializing Mime Type deny list"); - final boolean enableAudioTags = this.getConfigBool("parser.enableAudioTags", false); - this.log.config("Parser: parser.enableAudioTags= "+enableAudioTags); - final Set denyExt = this.getConfigSet(SwitchboardConstants.PARSER_EXTENSIONS_DENY); - final Set denyMime = this.getConfigSet(SwitchboardConstants.PARSER_MIME_DENY); + final boolean enableAudioTags = this.getConfigBool("parser.enableAudioTags", false); + this.log.config("Parser: parser.enableAudioTags= "+enableAudioTags); + final Set denyExt = this.getConfigSet(SwitchboardConstants.PARSER_EXTENSIONS_DENY); + final Set denyMime = this.getConfigSet(SwitchboardConstants.PARSER_MIME_DENY); - /* audioTagParser is disabled by default as it needs a temporary file (because of the JAudiotagger implementation) for each parsed document */ - if (!enableAudioTags) { - denyExt.addAll(audioTagParser.SupportedAudioFormat.getAllFileExtensions()); - denyMime.addAll(audioTagParser.SupportedAudioFormat.getAllMediaTypes()); + /* audioTagParser is disabled by default as it needs a temporary file (because of the JAudiotagger implementation) for each parsed document */ + if (!enableAudioTags) { + denyExt.addAll(audioTagParser.SupportedAudioFormat.getAllFileExtensions()); + denyMime.addAll(audioTagParser.SupportedAudioFormat.getAllMediaTypes()); - this.setConfig(SwitchboardConstants.PARSER_EXTENSIONS_DENY, denyExt); - this.setConfig(SwitchboardConstants.PARSER_MIME_DENY, denyMime); - this.setConfig("parser.enableAudioTags", true); - } + this.setConfig(SwitchboardConstants.PARSER_EXTENSIONS_DENY, denyExt); + this.setConfig(SwitchboardConstants.PARSER_MIME_DENY, denyMime); + this.setConfig("parser.enableAudioTags", true); + } - TextParser.setDenyMime(this.getConfig(SwitchboardConstants.PARSER_MIME_DENY, "")); - TextParser.setDenyExtension(this.getConfig(SwitchboardConstants.PARSER_EXTENSIONS_DENY, "")); + TextParser.setDenyMime(this.getConfig(SwitchboardConstants.PARSER_MIME_DENY, "")); + TextParser.setDenyExtension(this.getConfig(SwitchboardConstants.PARSER_EXTENSIONS_DENY, "")); - // start a loader - this.log.config("Starting Crawl Loader"); - this.loader = new LoaderDispatcher(this); + // start a loader + this.log.config("Starting Crawl Loader"); + this.loader = new LoaderDispatcher(this); - // load the robots.txt db - this.log.config("Initializing robots.txt DB"); - this.robots = new RobotsTxt(this.tables, this.loader, - this.getConfigInt(SwitchboardConstants.ROBOTS_TXT_THREADS_ACTIVE_MAX, SwitchboardConstants.ROBOTS_TXT_THREADS_ACTIVE_MAX_DEFAULT)); + // load donation frame + new Thread("yacy.importDonationIFrame") { + @Override + public void run() { + final ClientIdentification.Agent agent = ClientIdentification.getAgent(ClientIdentification.yacyInternetCrawlerAgentName); try { - this.log.config("Loaded robots.txt DB: " + this.robots.size() + " entries"); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } + final Response documentResponse = Switchboard.this.loader.load(Switchboard.this.loader.request(new DigestURL(getConfig("donation.iframesource", "")), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, agent); + if (documentResponse != null) FileUtils.copy(documentResponse.getContent(), new File(Switchboard.this.htDocsPath, getConfig("donation.iframetarget", ""))); + } catch (final Exception e) {} + } + }.start(); - // load oai tables - final Map oaiFriends = - OAIListFriendsLoader.loadListFriendsSources( - new File(appPath, "defaults/oaiListFriendsSource.xml"), - this.getDataPath()); - OAIListFriendsLoader.init(this.loader, oaiFriends, ClientIdentification.yacyInternetCrawlerAgent); - this.crawlQueues = new CrawlQueues(this, this.queuesRoot); - - // on startup, resume all crawls - this.setConfig(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL + "_isPaused", "false"); - this.setConfig(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL + "_isPaused_cause", ""); - this.setConfig(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL + "_isPaused", "false"); - this.setConfig(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL + "_isPaused_cause", ""); - this.setConfig(SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER + "_isPaused", "false"); - this.setConfig(SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER + "_isPaused_cause", ""); - this.crawlJobsStatus.put(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL, new Object[] {new Object(), false}); - this.crawlJobsStatus.put(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL, new Object[] {new Object(), false}); - this.crawlJobsStatus.put(SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER, new Object[] {new Object(), false}); - - // init cookie-Monitor - this.log.config("Starting Cookie Monitor"); - this.outgoingCookies = new ConcurrentHashMap<>(); - this.incomingCookies = new ConcurrentHashMap<>(); - - // init search history trackers - this.localSearchTracker = new ConcurrentHashMap<>(); // String:TreeSet - IP:set of Long(accessTime) - this.remoteSearchTracker = new ConcurrentHashMap<>(); - - // init messages: clean up message symbol - final File notifierSource = - new File(this.getAppPath(), this.getConfig( - SwitchboardConstants.HTROOT_PATH, - SwitchboardConstants.HTROOT_PATH_DEFAULT) + "/env/grafics/empty.gif"); - final File notifierDest = - new File( - this.getDataPath(SwitchboardConstants.HTDOCS_PATH, SwitchboardConstants.HTDOCS_PATH_DEFAULT), - "notifier.gif"); - try { - Files.copy(notifierSource, notifierDest); - } catch (final IOException e ) { - } + // load oai tables + final Map oaiFriends = + OAIListFriendsLoader.loadListFriendsSources( + new File(appPath, "defaults/oaiListFriendsSource.xml"), + this.getDataPath()); + OAIListFriendsLoader.init(this.loader, oaiFriends, ClientIdentification.yacyInternetCrawlerAgent); + + // load the robots.txt db + this.log.config("Initializing robots.txt DB"); + this.robots = new RobotsTxt(this.tables, this.loader, + this.getConfigInt(SwitchboardConstants.ROBOTS_TXT_THREADS_ACTIVE_MAX, SwitchboardConstants.ROBOTS_TXT_THREADS_ACTIVE_MAX_DEFAULT)); + try { + this.log.config("Loaded robots.txt DB: " + this.robots.size() + " entries"); + } catch (final IOException e) { + ConcurrentLog.logException(e); + } + this.crawlQueues = new CrawlQueues(this, this.queuesRoot); + + // on startup, resume all crawls + this.setConfig(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL + "_isPaused", "false"); + this.setConfig(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL + "_isPaused_cause", ""); + this.setConfig(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL + "_isPaused", "false"); + this.setConfig(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL + "_isPaused_cause", ""); + this.setConfig(SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER + "_isPaused", "false"); + this.setConfig(SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER + "_isPaused_cause", ""); + this.crawlJobsStatus.put(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL, new Object[] {new Object(), false}); + this.crawlJobsStatus.put(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL, new Object[] {new Object(), false}); + this.crawlJobsStatus.put(SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER, new Object[] {new Object(), false}); + + // init cookie-Monitor + this.log.config("Starting Cookie Monitor"); + this.outgoingCookies = new ConcurrentHashMap<>(); + this.incomingCookies = new ConcurrentHashMap<>(); + + // init search history trackers + this.localSearchTracker = new ConcurrentHashMap<>(); // String:TreeSet - IP:set of Long(accessTime) + this.remoteSearchTracker = new ConcurrentHashMap<>(); + + // init messages: clean up message symbol + final File notifierSource = new File(this.htRootPath.getAbsolutePath() + "/env/grafics/empty.gif"); + final File notifierDest = new File(this.htDocsPath, "notifier.gif"); + try { + Files.copy(notifierSource, notifierDest); + } catch (final IOException e ) { + } - // init nameCacheNoCachingList - try { - Domains.setNoCachingPatterns(this.getConfig(SwitchboardConstants.HTTPC_NAME_CACHE_CACHING_PATTERNS_NO, "")); - } catch (final PatternSyntaxException pse) { - ConcurrentLog.severe("Switchboard", "Invalid regular expression in " - + SwitchboardConstants.HTTPC_NAME_CACHE_CACHING_PATTERNS_NO - + " property: " + pse.getMessage()); - System.exit(-1); - } + // init nameCacheNoCachingList + try { + Domains.setNoCachingPatterns(this.getConfig(SwitchboardConstants.HTTPC_NAME_CACHE_CACHING_PATTERNS_NO, "")); + } catch (final PatternSyntaxException pse) { + ConcurrentLog.severe("Switchboard", "Invalid regular expression in " + + SwitchboardConstants.HTTPC_NAME_CACHE_CACHING_PATTERNS_NO + + " property: " + pse.getMessage()); + System.exit(-1); + } - // generate snippets cache - this.log.config("Initializing Snippet Cache"); + // generate snippets cache + this.log.config("Initializing Snippet Cache"); - TextSnippet.statistics.setEnabled(this.getConfigBool(SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED, - SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED_DEFAULT)); + TextSnippet.statistics.setEnabled(this.getConfigBool(SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED, + SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED_DEFAULT)); - // init the wiki - wikiParser = new WikiCode(); + // init the wiki + wikiParser = new WikiCode(); - // initializing the resourceObserver - this.observer = new ResourceObserver(this); + // initializing the resourceObserver + this.observer = new ResourceObserver(this); - final ResourceObserver resourceObserver = this.observer; - new OneTimeBusyThread("ResourceObserver.resourceObserverJob") { + final ResourceObserver resourceObserver = this.observer; + new OneTimeBusyThread("ResourceObserver.resourceObserverJob") { - @Override - public boolean jobImpl() throws Exception { - resourceObserver.resourceObserverJob(); - return true; - } - }.start(); - - // initializing the stackCrawlThread - this.crawlStacker = - new CrawlStacker( - this.robots, - this.crawlQueues, - this.crawler, - this.index, - this.peers, - this.isIntranetMode(), - this.isGlobalMode(), - this.domainList); // Intranet and Global mode may be both true! - - // possibly switch off localIP check - Domains.setNoLocalCheck(this.isAllIPMode()); - - // check status of account configuration: when local url crawling is allowed, it is not allowed - // that an automatic authorization of localhost is done, because in this case crawls from local - // addresses are blocked to prevent attack szenarios where remote pages contain links to localhost - // addresses that can steer a YaCy peer - if ( !this.getConfigBool(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, false) ) { - if ( this.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").startsWith("0000") ) { - // the password was set automatically with a random value. - // We must remove that here to prevent that a user cannot log in any more - this.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""); - // after this a message must be generated to alert the user to set a new password - this.log.info("RANDOM PASSWORD REMOVED! User must set a new password"); - } - } + @Override + public boolean jobImpl() throws Exception { + resourceObserver.resourceObserverJob(); + return true; + } + }.start(); - // initializing dht chunk generation - this.dhtMaxReferenceCount = (int) this.getConfigLong(SwitchboardConstants.INDEX_DIST_CHUNK_SIZE_START, 50); - - // init robinson cluster - // before we do that, we wait some time until the seed list is loaded. - this.clusterhashes = this.peers.clusterHashes(this.getConfig("cluster.peers.yacydomain", "")); - - // deploy blocking threads - this.indexingStorageProcessor = - new WorkflowProcessor<>( - "storeDocumentIndex", - "This is the sequencing step of the indexing queue. Files are written as streams, too much councurrency would destroy IO performance. In this process the words are written to the RWI cache, which flushes if it is full.", - new String[] { - "RWI/Cache/Collections" - }, - in -> { - Switchboard.this.storeDocumentIndex(in); - return null; - }, - 2, - null, - 1); - this.indexingAnalysisProcessor = - new WorkflowProcessor<>( - "webStructureAnalysis", - "This just stores the link structure of the document into a web structure database.", - new String[] { - "storeDocumentIndex" - }, - in -> Switchboard.this.webStructureAnalysis(in), - WorkflowProcessor.availableCPU + 1, - this.indexingStorageProcessor, - WorkflowProcessor.availableCPU); - this.indexingCondensementProcessor = - new WorkflowProcessor<>( - "condenseDocument", - "This does a structural analysis of plain texts: markup of headlines, slicing into phrases (i.e. sentences), markup with position, counting of words, calculation of term frequency.", - new String[] { - "webStructureAnalysis" - }, - in -> Switchboard.this.condenseDocument(in), - WorkflowProcessor.availableCPU + 1, - this.indexingAnalysisProcessor, - WorkflowProcessor.availableCPU); - this.indexingDocumentProcessor = - new WorkflowProcessor<>( - "parseDocument", - "This does the parsing of the newly loaded documents from the web. The result is not only a plain text document, but also a list of URLs that are embedded into the document. The urls are handed over to the CrawlStacker. This process has two child process queues!", - new String[] { - "condenseDocument", "CrawlStacker" - }, - in -> Switchboard.this.parseDocument(in), - Math.max(20, WorkflowProcessor.availableCPU * 2), // it may happen that this is filled with new files from the search process. That means there should be enough place for two result pages - this.indexingCondensementProcessor, - WorkflowProcessor.availableCPU); - - // deploy busy threads - this.log.config("Starting Threads"); - MemoryControl.gc(10000, "plasmaSwitchboard, help for profiler"); // help for profiler - thq + // initializing the stackCrawlThread + this.crawlStacker = + new CrawlStacker( + this.robots, + this.crawlQueues, + this.crawler, + this.index, + this.peers, + this.isIntranetMode(), + this.isGlobalMode(), + this.domainList); // Intranet and Global mode may be both true! + + // possibly switch off localIP check + Domains.setNoLocalCheck(this.isAllIPMode()); + + // check status of account configuration: when local url crawling is allowed, it is not allowed + // that an automatic authorization of localhost is done, because in this case crawls from local + // addresses are blocked to prevent attack szenarios where remote pages contain links to localhost + // addresses that can steer a YaCy peer + if ( !this.getConfigBool(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, false) ) { + if ( this.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").startsWith("0000") ) { + // the password was set automatically with a random value. + // We must remove that here to prevent that a user cannot log in any more + this.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""); + // after this a message must be generated to alert the user to set a new password + this.log.info("RANDOM PASSWORD REMOVED! User must set a new password"); + } + } - this.deployThread( - SwitchboardConstants.CLEANUP, - "Cleanup", - "cleaning process", - null, - new InstantBusyThread("Switchboard.cleanupJob", 30000, 10000) { + // initializing dht chunk generation + this.dhtMaxReferenceCount = (int) this.getConfigLong(SwitchboardConstants.INDEX_DIST_CHUNK_SIZE_START, 50); - @Override - public boolean jobImpl() throws Exception { - return Switchboard.this.cleanupJob(); - } + // init robinson cluster + // before we do that, we wait some time until the seed list is loaded. + this.clusterhashes = this.peers.clusterHashes(this.getConfig("cluster.peers.yacydomain", "")); - @Override - public int getJobCount() { - return Switchboard.this.cleanupJobSize(); - } + // deploy blocking threads + this.indexingStorageProcessor = + new WorkflowProcessor<>( + "storeDocumentIndex", + "This is the sequencing step of the indexing queue. Files are written as streams, too much councurrency would destroy IO performance. In this process the words are written to the RWI cache, which flushes if it is full.", + new String[] { + "RWI/Cache/Collections" + }, + in -> { + Switchboard.this.storeDocumentIndex(in); + return null; + }, + 2, + null, + 1); + this.indexingAnalysisProcessor = + new WorkflowProcessor<>( + "webStructureAnalysis", + "This just stores the link structure of the document into a web structure database.", + new String[] { + "storeDocumentIndex" + }, + in -> Switchboard.this.webStructureAnalysis(in), + WorkflowProcessor.availableCPU + 1, + this.indexingStorageProcessor, + WorkflowProcessor.availableCPU); + this.indexingCondensementProcessor = + new WorkflowProcessor<>( + "condenseDocument", + "This does a structural analysis of plain texts: markup of headlines, slicing into phrases (i.e. sentences), markup with position, counting of words, calculation of term frequency.", + new String[] { + "webStructureAnalysis" + }, + in -> Switchboard.this.condenseDocument(in), + WorkflowProcessor.availableCPU + 1, + this.indexingAnalysisProcessor, + WorkflowProcessor.availableCPU); + this.indexingDocumentProcessor = + new WorkflowProcessor<>( + "parseDocument", + "This does the parsing of the newly loaded documents from the web. The result is not only a plain text document, but also a list of URLs that are embedded into the document. The urls are handed over to the CrawlStacker. This process has two child process queues!", + new String[] { + "condenseDocument", "CrawlStacker" + }, + in -> Switchboard.this.parseDocument(in), + Math.max(20, WorkflowProcessor.availableCPU * 2), // it may happen that this is filled with new files from the search process. That means there should be enough place for two result pages + this.indexingCondensementProcessor, + WorkflowProcessor.availableCPU); + + // deploy busy threads + this.log.config("Starting Threads"); + MemoryControl.gc(10000, "plasmaSwitchboard, help for profiler"); // help for profiler - thq + + this.deployThread( + SwitchboardConstants.CLEANUP, + "Cleanup", + "cleaning process", + null, + new InstantBusyThread("Switchboard.cleanupJob", 30000, 10000) { - @Override - public void freememImpl() { - } + @Override + public boolean jobImpl() throws Exception { + return Switchboard.this.cleanupJob(); + } - }, - 60000); // all 10 minutes, wait 1 minute until first run + @Override + public int getJobCount() { + return Switchboard.this.cleanupJobSize(); + } - this.deployThread( - SwitchboardConstants.SCHEDULER, - "Scheduler", - "starts scheduled processes from the API Processing table", - null, - new InstantBusyThread("Switchboard.schedulerJob", 30000, 10000) { - @Override - public boolean jobImpl() throws Exception { - return Switchboard.this.schedulerJob(); - } + @Override + public void freememImpl() { + } - @Override - public int getJobCount() { - return Switchboard.this.schedulerJobSize(); - } + }, + 60000); // all 10 minutes, wait 1 minute until first run - @Override - public void freememImpl() { - } - }, - 60000); // all 10 minutes, wait 1 minute until first run + this.deployThread( + SwitchboardConstants.SCHEDULER, + "Scheduler", + "starts scheduled processes from the API Processing table", + null, + new InstantBusyThread("Switchboard.schedulerJob", 30000, 10000) { + @Override + public boolean jobImpl() throws Exception { + return Switchboard.this.schedulerJob(); + } - this.deployThread( - SwitchboardConstants.SURROGATES, - "Surrogates", - "A thread that polls the SURROGATES path and puts all Documents in one surroagte file into the indexing queue.", - null, - new InstantBusyThread("Switchboard.surrogateProcess", 20000, 0) { - @Override - public boolean jobImpl() throws Exception { - return Switchboard.this.surrogateProcess(); - } + @Override + public int getJobCount() { + return Switchboard.this.schedulerJobSize(); + } - @Override - public int getJobCount() { - return Switchboard.this.surrogateQueueSize(); - } + @Override + public void freememImpl() { + } + }, + 60000); // all 10 minutes, wait 1 minute until first run - @Override - public void freememImpl() { - Switchboard.this.surrogateFreeMem(); - } - }, - 10000); + this.deployThread( + SwitchboardConstants.SURROGATES, + "Surrogates", + "A thread that polls the SURROGATES path and puts all Documents in one surroagte file into the indexing queue.", + null, + new InstantBusyThread("Switchboard.surrogateProcess", 20000, 0) { + @Override + public boolean jobImpl() throws Exception { + return Switchboard.this.surrogateProcess(); + } - this.initRemoteCrawler(this.getConfigBool(SwitchboardConstants.CRAWLJOB_REMOTE, false)); - this.initAutocrawl(this.getConfigBool(SwitchboardConstants.AUTOCRAWL, false)); + @Override + public int getJobCount() { + return Switchboard.this.surrogateQueueSize(); + } - final CrawlQueues crawlQueue = this.crawlQueues; - this.deployThread( - SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL, - "Local Crawl", - "thread that performes a single crawl step from the local crawl queue", - "/IndexCreateQueues_p.html?stack=LOCAL", - new InstantBusyThread("CrawlQueues.coreCrawlJob", 0, 0) { - @Override - public boolean jobImpl() throws Exception { - return crawlQueue.coreCrawlJob(); - } + @Override + public void freememImpl() { + Switchboard.this.surrogateFreeMem(); + } + }, + 10000); + + this.initRemoteCrawler(this.getConfigBool(SwitchboardConstants.CRAWLJOB_REMOTE, false)); + this.initAutocrawl(this.getConfigBool(SwitchboardConstants.AUTOCRAWL, false)); + + final CrawlQueues crawlQueue = this.crawlQueues; + this.deployThread( + SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL, + "Local Crawl", + "thread that performes a single crawl step from the local crawl queue", + "/IndexCreateQueues_p.html?stack=LOCAL", + new InstantBusyThread("CrawlQueues.coreCrawlJob", 0, 0) { + @Override + public boolean jobImpl() throws Exception { + return crawlQueue.coreCrawlJob(); + } - @Override - public int getJobCount() { - return crawlQueue.coreCrawlJobSize(); - } + @Override + public int getJobCount() { + return crawlQueue.coreCrawlJobSize(); + } - @Override - public void freememImpl() { - crawlQueue.freemem(); - } - }, - 10000); + @Override + public void freememImpl() { + crawlQueue.freemem(); + } + }, + 10000); + + final Network net = this.yc; + this.deployThread( + SwitchboardConstants.SEED_UPLOAD, + "Seed-List Upload", + "task that a principal peer performes to generate and upload a seed-list to a ftp account", + null, + new InstantBusyThread("Network.publishSeedList", 600000, 300000) { + @Override + public boolean jobImpl() throws Exception { + net.publishSeedList(); + return true; + } + }, + 180000); - final Network net = this.yc; - this.deployThread( - SwitchboardConstants.SEED_UPLOAD, - "Seed-List Upload", - "task that a principal peer performes to generate and upload a seed-list to a ftp account", - null, - new InstantBusyThread("Network.publishSeedList", 600000, 300000) { - @Override - public boolean jobImpl() throws Exception { - net.publishSeedList(); - return true; - } - }, - 180000); + this.deployThread( + SwitchboardConstants.PEER_PING, + "YaCy Core", + "this is the p2p-control and peer-ping task", + null, + new InstantBusyThread("Network.peerPing", 30000, 30000) { + @Override + public boolean jobImpl() throws Exception { + net.peerPing(); + return true; + } + }, + 10000); + this.deployThread( + SwitchboardConstants.INDEX_DIST, + "DHT Distribution", + "selection, transfer and deletion of index entries that are not searched on your peer, but on others", + null, + new InstantBusyThread("Switchboard.dhtTransferJob", 10000, 1000) { + @Override + public boolean jobImpl() throws Exception { + return Switchboard.this.dhtTransferJob(); + } + }, + 60000, + Long.parseLong(this.getConfig(SwitchboardConstants.INDEX_DIST_IDLESLEEP, "5000")), + Long.parseLong(this.getConfig(SwitchboardConstants.INDEX_DIST_BUSYSLEEP, "0")), + Long.parseLong(this.getConfig(SwitchboardConstants.INDEX_DIST_MEMPREREQ, "1000000")), + Double.parseDouble(this.getConfig(SwitchboardConstants.INDEX_DIST_LOADPREREQ, "9.0"))); - this.deployThread( - SwitchboardConstants.PEER_PING, - "YaCy Core", - "this is the p2p-control and peer-ping task", - null, - new InstantBusyThread("Network.peerPing", 30000, 30000) { - @Override - public boolean jobImpl() throws Exception { - net.peerPing(); - return true; - } - }, - 10000); - this.deployThread( - SwitchboardConstants.INDEX_DIST, - "DHT Distribution", - "selection, transfer and deletion of index entries that are not searched on your peer, but on others", - null, - new InstantBusyThread("Switchboard.dhtTransferJob", 10000, 1000) { - @Override - public boolean jobImpl() throws Exception { - return Switchboard.this.dhtTransferJob(); - } - }, - 60000, - Long.parseLong(this.getConfig(SwitchboardConstants.INDEX_DIST_IDLESLEEP, "5000")), - Long.parseLong(this.getConfig(SwitchboardConstants.INDEX_DIST_BUSYSLEEP, "0")), - Long.parseLong(this.getConfig(SwitchboardConstants.INDEX_DIST_MEMPREREQ, "1000000")), - Double.parseDouble(this.getConfig(SwitchboardConstants.INDEX_DIST_LOADPREREQ, "9.0"))); - - // set network-specific performance attributes - if ( this.firstInit ) { - this.setRemotecrawlPPM(Math.max(1, (int) this.getConfigLong("network.unit.remotecrawl.speed", 60))); - } + // set network-specific performance attributes + if ( this.firstInit ) { + this.setRemotecrawlPPM(Math.max(1, (int) this.getConfigLong("network.unit.remotecrawl.speed", 60))); + } - // test routine for snippet fetch - //Set query = new HashSet(); - //query.add(CrawlSwitchboardEntry.word2hash("Weitergabe")); - //query.add(CrawlSwitchboardEntry.word2hash("Zahl")); - //plasmaSnippetCache.result scr = snippetCache.retrieve(new URL("http://www.heise.de/mobil/newsticker/meldung/mail/54980"), query, true); - //plasmaSnippetCache.result scr = snippetCache.retrieve(new URL("http://www.heise.de/security/news/foren/go.shtml?read=1&msg_id=7301419&forum_id=72721"), query, true); - //plasmaSnippetCache.result scr = snippetCache.retrieve(new URL("http://www.heise.de/kiosk/archiv/ct/2003/4/20"), query, true, 260); + // test routine for snippet fetch + //Set query = new HashSet(); + //query.add(CrawlSwitchboardEntry.word2hash("Weitergabe")); + //query.add(CrawlSwitchboardEntry.word2hash("Zahl")); + //plasmaSnippetCache.result scr = snippetCache.retrieve(new URL("http://www.heise.de/mobil/newsticker/meldung/mail/54980"), query, true); + //plasmaSnippetCache.result scr = snippetCache.retrieve(new URL("http://www.heise.de/security/news/foren/go.shtml?read=1&msg_id=7301419&forum_id=72721"), query, true); + //plasmaSnippetCache.result scr = snippetCache.retrieve(new URL("http://www.heise.de/kiosk/archiv/ct/2003/4/20"), query, true, 260); - this.trail = new LinkedBlockingQueue<>(); + this.trail = new LinkedBlockingQueue<>(); - this.log.config("Finished Switchboard Initialization"); + this.log.config("Finished Switchboard Initialization"); } /** diff --git a/source/net/yacy/search/index/ErrorCache.java b/source/net/yacy/search/index/ErrorCache.java index 3289c6655a..bc0ac6751b 100644 --- a/source/net/yacy/search/index/ErrorCache.java +++ b/source/net/yacy/search/index/ErrorCache.java @@ -1,7 +1,7 @@ /** * ErrorCache * Copyright 2013 by Michael Peter Christen - * First released 17.10.2013 at http://yacy.net + * First released 17.10.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/search/index/ReindexSolrBusyThread.java b/source/net/yacy/search/index/ReindexSolrBusyThread.java index 4be6d6e95e..40f66b2b55 100644 --- a/source/net/yacy/search/index/ReindexSolrBusyThread.java +++ b/source/net/yacy/search/index/ReindexSolrBusyThread.java @@ -2,7 +2,7 @@ /** * ReindexSolrBusyThread * Copyright 2013 by Michael Peter Christen - * First released 13.05.2013 at http://yacy.net + * First released 13.05.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/search/query/AccessTracker.java b/source/net/yacy/search/query/AccessTracker.java index df327fd26c..4da4c16625 100644 --- a/source/net/yacy/search/query/AccessTracker.java +++ b/source/net/yacy/search/query/AccessTracker.java @@ -2,7 +2,7 @@ * AccessTracker * an interface for Adaptive Replacement Caches * Copyright 2009 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 29.08.2009 at http://yacy.net + * First released 29.08.2009 at https://yacy.net * * $LastChangedDate$ * $LastChangedRevision$ diff --git a/source/net/yacy/search/query/HeuristicResult.java b/source/net/yacy/search/query/HeuristicResult.java index 34b2f4106e..5f9050361c 100644 --- a/source/net/yacy/search/query/HeuristicResult.java +++ b/source/net/yacy/search/query/HeuristicResult.java @@ -1,7 +1,7 @@ /** * HeuristicResult * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 01.11.2012 at http://yacy.net + * First released 01.11.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/search/query/SearchEventType.java b/source/net/yacy/search/query/SearchEventType.java index 9276757952..fdfdafecf8 100644 --- a/source/net/yacy/search/query/SearchEventType.java +++ b/source/net/yacy/search/query/SearchEventType.java @@ -1,7 +1,7 @@ /** * SearchEventType * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany - * First released 01.11.2012 at http://yacy.net + * First released 01.11.2012 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/search/ranking/BlockRank.java b/source/net/yacy/search/ranking/BlockRank.java index 04371664fd..9ccf93ce34 100644 --- a/source/net/yacy/search/ranking/BlockRank.java +++ b/source/net/yacy/search/ranking/BlockRank.java @@ -1,7 +1,7 @@ /** * BlockRankCollector * Copyright 2011 by Michael Christen - * First released 18.05.2011 at http://yacy.net + * First released 18.05.2011 at https://yacy.net * * $LastChangedDate: 2011-04-26 19:39:16 +0200 (Di, 26 Apr 2011) $ * $LastChangedRevision: 7676 $ diff --git a/source/net/yacy/search/schema/CollectionConfiguration.java b/source/net/yacy/search/schema/CollectionConfiguration.java index 60655ba0f7..1e5c5b2f6f 100644 --- a/source/net/yacy/search/schema/CollectionConfiguration.java +++ b/source/net/yacy/search/schema/CollectionConfiguration.java @@ -1,7 +1,7 @@ /** * CollectionConfiguration * Copyright 2011 by Michael Peter Christen - * First released 14.04.2011 at http://yacy.net + * First released 14.04.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 22:05:04 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7654 $ diff --git a/source/net/yacy/search/schema/CollectionSchema.java b/source/net/yacy/search/schema/CollectionSchema.java index 0fa5091846..4b9e1d8b9b 100644 --- a/source/net/yacy/search/schema/CollectionSchema.java +++ b/source/net/yacy/search/schema/CollectionSchema.java @@ -1,7 +1,7 @@ /** * CollectionSchema * Copyright 2011 by Michael Peter Christen - * First released 14.04.2011 at http://yacy.net + * First released 14.04.2011 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/search/schema/HyperlinkEdge.java b/source/net/yacy/search/schema/HyperlinkEdge.java index 5131ed37c2..ef3eceb066 100644 --- a/source/net/yacy/search/schema/HyperlinkEdge.java +++ b/source/net/yacy/search/schema/HyperlinkEdge.java @@ -1,7 +1,7 @@ /** * HyperlinkEdge * Copyright 2014 by Michael Peter Christen - * First released 04.04.2014 at http://yacy.net + * First released 04.04.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/search/schema/HyperlinkGraph.java b/source/net/yacy/search/schema/HyperlinkGraph.java index 9bd4d5d4b4..8068992cbd 100644 --- a/source/net/yacy/search/schema/HyperlinkGraph.java +++ b/source/net/yacy/search/schema/HyperlinkGraph.java @@ -1,7 +1,7 @@ /** * HyperlinkGraph * Copyright 2014 by Michael Peter Christen - * First released 08.04.2014 at http://yacy.net + * First released 08.04.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/search/schema/HyperlinkType.java b/source/net/yacy/search/schema/HyperlinkType.java index 9c4c941ed2..7d80d89e07 100644 --- a/source/net/yacy/search/schema/HyperlinkType.java +++ b/source/net/yacy/search/schema/HyperlinkType.java @@ -1,7 +1,7 @@ /** * HyperlinkType * Copyright 2014 by Michael Peter Christen - * First released 04.04.2014 at http://yacy.net + * First released 04.04.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/search/schema/WebgraphConfiguration.java b/source/net/yacy/search/schema/WebgraphConfiguration.java index 7ee09e4d52..2cc6052489 100644 --- a/source/net/yacy/search/schema/WebgraphConfiguration.java +++ b/source/net/yacy/search/schema/WebgraphConfiguration.java @@ -1,7 +1,7 @@ /** * WebgraphConfiguration * Copyright 2011 by Michael Peter Christen - * First released 14.04.2011 at http://yacy.net + * First released 14.04.2011 at https://yacy.net * * $LastChangedDate: 2011-04-14 22:05:04 +0200 (Do, 14 Apr 2011) $ * $LastChangedRevision: 7654 $ diff --git a/source/net/yacy/search/schema/WebgraphSchema.java b/source/net/yacy/search/schema/WebgraphSchema.java index bd16245a12..2d45ed195e 100644 --- a/source/net/yacy/search/schema/WebgraphSchema.java +++ b/source/net/yacy/search/schema/WebgraphSchema.java @@ -1,7 +1,7 @@ /** * WebgraphSchema * Copyright 2011 by Michael Peter Christen - * First released 19.02.2013 at http://yacy.net + * First released 19.02.2013 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/visualization/AnimationGIF.java b/source/net/yacy/visualization/AnimationGIF.java index 78ecc4cb69..d39df36e07 100644 --- a/source/net/yacy/visualization/AnimationGIF.java +++ b/source/net/yacy/visualization/AnimationGIF.java @@ -1,7 +1,7 @@ /** * AnimationGIF * Copyright 2010 by Michael Christen - * First released 20.11.2010 at http://yacy.net + * First released 20.11.2010 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/visualization/AnimationPlotter.java b/source/net/yacy/visualization/AnimationPlotter.java index 565fc7019e..1c46c25bec 100644 --- a/source/net/yacy/visualization/AnimationPlotter.java +++ b/source/net/yacy/visualization/AnimationPlotter.java @@ -1,7 +1,7 @@ /** * AnimationPlotter * Copyright 2013 by Michael Christen - * First released 9.9.2010 at http://yacy.net + * First released 9.9.2010 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/visualization/ChartPlotter.java b/source/net/yacy/visualization/ChartPlotter.java index 2eae51860b..603ac19878 100644 --- a/source/net/yacy/visualization/ChartPlotter.java +++ b/source/net/yacy/visualization/ChartPlotter.java @@ -1,7 +1,7 @@ /** * ChartPlotter * Copyright 2005 by Michael Christen - * First released 26.10.2005 at http://yacy.net + * First released 26.10.2005 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/visualization/CircleTool.java b/source/net/yacy/visualization/CircleTool.java index def9c6d51c..556396a7f5 100644 --- a/source/net/yacy/visualization/CircleTool.java +++ b/source/net/yacy/visualization/CircleTool.java @@ -1,7 +1,7 @@ /** * CircleTool * Copyright 2007 by Michael Christen - * First released 22.05.2007 at http://yacy.net + * First released 22.05.2007 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/visualization/FontGenerator3Pixel.java b/source/net/yacy/visualization/FontGenerator3Pixel.java index 046085a2dc..89c1c1a36a 100644 --- a/source/net/yacy/visualization/FontGenerator3Pixel.java +++ b/source/net/yacy/visualization/FontGenerator3Pixel.java @@ -1,7 +1,7 @@ /** * FontGenerator3Pixel * Copyright 2014 by Michael Christen - * First released 14.02.2014 at http://yacy.net + * First released 14.02.2014 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/visualization/FontGenerator5Pixel.java b/source/net/yacy/visualization/FontGenerator5Pixel.java index c8cae68490..7d89cccbd6 100644 --- a/source/net/yacy/visualization/FontGenerator5Pixel.java +++ b/source/net/yacy/visualization/FontGenerator5Pixel.java @@ -1,7 +1,7 @@ /** * FontGenerator5Pixel * Copyright 2005 by Michael Christen - * First released 31.10.2005 at http://yacy.net + * First released 31.10.2005 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/visualization/GraphPlotter.java b/source/net/yacy/visualization/GraphPlotter.java index ff537da710..6f399157ba 100644 --- a/source/net/yacy/visualization/GraphPlotter.java +++ b/source/net/yacy/visualization/GraphPlotter.java @@ -1,7 +1,7 @@ /** * GraphPlotter * Copyright 2007 by Michael Christen - * First released 22.05.2007 at http://yacy.net + * First released 22.05.2007 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/visualization/HexGridPlotter.java b/source/net/yacy/visualization/HexGridPlotter.java index 4269a5dce2..17ff08b2e7 100644 --- a/source/net/yacy/visualization/HexGridPlotter.java +++ b/source/net/yacy/visualization/HexGridPlotter.java @@ -1,7 +1,7 @@ /** * HexGridPlotter * Copyright 2011 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * First released 18.02.2010 at http://yacy.net + * First released 18.02.2010 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/visualization/PrintTool.java b/source/net/yacy/visualization/PrintTool.java index 2b53352cb9..6bbfa96b5d 100644 --- a/source/net/yacy/visualization/PrintTool.java +++ b/source/net/yacy/visualization/PrintTool.java @@ -1,7 +1,7 @@ /** * PrintTool * Copyright 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * First released 22.05.2007 at http://yacy.net + * First released 22.05.2007 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/visualization/RasterPlotter.java b/source/net/yacy/visualization/RasterPlotter.java index 91a78cf2f3..f7756a4ffe 100644 --- a/source/net/yacy/visualization/RasterPlotter.java +++ b/source/net/yacy/visualization/RasterPlotter.java @@ -1,7 +1,7 @@ /** * RasterPlotter * Copyright 2005 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany - * First released 16.09.2005 at http://yacy.net + * First released 16.09.2005 at https://yacy.net * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index fdf24519d1..a7626bac3f 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -35,8 +35,6 @@ import java.io.InputStreamReader; import java.io.PrintWriter; import java.io.RandomAccessFile; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.util.ArrayList; @@ -58,7 +56,6 @@ import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.solr.instance.RemoteInstance; -import net.yacy.cora.plugin.ClassProvider; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.cora.protocol.HeaderFramework; @@ -196,7 +193,7 @@ private static void startup(final File dataHome, final File appHome, final long f = new File(dataHome, "DATA/yacy.running"); if (!f.createNewFile()) ConcurrentLog.severe("STARTUP", "WARNING: the file " + f + " can not be created!"); try { - FileOutputStream fos = new FileOutputStream(f); + final FileOutputStream fos = new FileOutputStream(f); fos.write(Integer.toString(OS.getPID()).getBytes()); fos.close(); } catch (final Exception e) { } // write PID @@ -204,7 +201,7 @@ private static void startup(final File dataHome, final File appHome, final long FileChannel channel = null; FileLock lock = null; try { - RandomAccessFile raf = new RandomAccessFile(f,"rw"); + final RandomAccessFile raf = new RandomAccessFile(f,"rw"); channel = raf.getChannel(); lock = channel.tryLock(); // lock yacy.running raf.close(); @@ -435,7 +432,7 @@ private static void delete(final File f) { * @see File#mkdir() * @param path */ - private static void mkdirIfNeseccary(final File path) { + public static void mkdirIfNeseccary(final File path) { if (!(path.exists())) if(!path.mkdir()) ConcurrentLog.warn("STARTUP", "could not create directory "+ path.toString()); diff --git a/startYACY.sh b/startYACY.sh index 33ebefc510..3ee15d88bf 100755 --- a/startYACY.sh +++ b/startYACY.sh @@ -247,7 +247,7 @@ if [ $PRINTONLY -eq 1 ];then else echo "****************** YaCy Web Crawler/Indexer & Search Engine *******************" echo "**** (C) by Michael Peter Christen, usage granted under the GPL Version 2 ****" - echo "**** USE AT YOUR OWN RISK! Project home and releases: http://yacy.net/ ****" + echo "**** USE AT YOUR OWN RISK! Project home and releases: https://yacy.net/ ****" echo "** LOG of YaCy: DATA/LOG/yacy00.log (and yacy.log) **" echo "** STOP YaCy: execute stopYACY.sh and wait some seconds **" echo "** GET HELP for YaCy: join our community at https://community.searchlab.eu **" diff --git a/startYACY_debug.bat b/startYACY_debug.bat index 73b705e8c1..1ff4ae8170 100644 --- a/startYACY_debug.bat +++ b/startYACY_debug.bat @@ -26,7 +26,7 @@ Echo Priority:%priority% Echo ****************** YaCy Web Crawler/Indexer ^& Search Engine ****************** Echo **** (C) by Michael Peter Christen, usage granted under the GPL Version 2 **** -Echo **** USE AT YOUR OWN RISK! Project home and releases: http://yacy.net/ **** +Echo **** USE AT YOUR OWN RISK! Project home and releases: https://yacy.net/ **** Echo ** LOG of YaCy: DATA/LOG/yacy00.log (and yacy^.log) ** Echo ** STOP YaCy: execute stopYACY.bat and wait some seconds ** Echo ** GET HELP for YaCy: join our community at https://community.searchlab.eu ** diff --git a/test/java/net/yacy/cora/document/id/MultiProtocolURLTest.java b/test/java/net/yacy/cora/document/id/MultiProtocolURLTest.java index c189cb67fb..4162096c96 100644 --- a/test/java/net/yacy/cora/document/id/MultiProtocolURLTest.java +++ b/test/java/net/yacy/cora/document/id/MultiProtocolURLTest.java @@ -230,11 +230,11 @@ public void testGetAttribute() throws Exception { // some test url/uri with problems in the past String[][] testStrings = new String[][]{ // teststring , expectedresultkey, expectedresultvalue - new String[]{"http://yacy.net?&test", "test", ""}, + new String[]{"https://yacy.net?&test", "test", ""}, /* Encoded UTF-8 2 bytes characters parameter value */ new String[]{"https://zh.wikipedia.org/w/index.php?search=encodedlatinchars%C3%A0%C3%A4%C3%A2%C3%A9%C3%A8%C3%AF%C3%AE%C3%B4%C3%B6%C3%B9", "search", "encodedlatincharsàäâéèïîôöù"}, /* Non encoded UTF-8 2 bytes characters parameter value */ - new String[]{"http://yacy.net?query=unencodedlatincharsàäâéèïîôöù", "query", "unencodedlatincharsàäâéèïîôöù"}, + new String[]{"https://yacy.net?query=unencodedlatincharsàäâéèïîôöù", "query", "unencodedlatincharsàäâéèïîôöù"}, /* Encoded UTF-8 3 bytes characters parameter value */ new String[]{"https://zh.wikipedia.org/w/index.php?query=%E6%96%B9%E9%87%9D%E8%88%87%E6%8C%87%E5%BC%95", "query", "方針與指引"}, /* Non encoded UTF-8 3 bytes characters parameter value */ @@ -312,12 +312,12 @@ public void testStaticToTokens() { public void testToTokens() throws MalformedURLException { String[][] testStrings = new String[][]{ // test string , "expected tokens" - new String[]{"http://yacy.net?&test", "yacy net test"}, + new String[]{"https://yacy.net?&test", "yacy net test"}, new String[]{"http://example.net/camelCased/subpath1/PATH_EXAMPLE", "example net camelCased subpath1 PATH EXAMPLE camel Cased subpath 1"}, /* Encoded UTF-8 2 bytes characters parameter value */ new String[]{"https://zh.wikipedia.org/w/index.php?search=encodedlatinchars%C3%A0%C3%A4%C3%A2%C3%A9%C3%A8%C3%AF%C3%AE%C3%B4%C3%B6%C3%B9", "zh wikipedia org w index php search encodedlatincharsàäâéèïîôöù"}, /* Non encoded UTF-8 2 bytes characters parameter value */ - new String[]{"http://yacy.net?query=unencodedlatincharsàäâéèïîôöù", "yacy net query unencodedlatincharsàäâéèïîôöù"}, + new String[]{"https://yacy.net?query=unencodedlatincharsàäâéèïîôöù", "yacy net query unencodedlatincharsàäâéèïîôöù"}, /* Encoded UTF-8 3 bytes characters parameter value */ new String[]{"https://zh.wikipedia.org/w/index.php?query=%E6%96%B9%E9%87%9D%E8%88%87%E6%8C%87%E5%BC%95", "zh wikipedia org w index php query 方針與指引"}, /* Non encoded UTF-8 3 bytes characters parameter value */ diff --git a/test/java/net/yacy/crawler/data/CacheTest.java b/test/java/net/yacy/crawler/data/CacheTest.java index 3feb29c23d..d1d25ad4b4 100644 --- a/test/java/net/yacy/crawler/data/CacheTest.java +++ b/test/java/net/yacy/crawler/data/CacheTest.java @@ -99,7 +99,7 @@ private static byte[] generateFlushableContent() { @Test public void testGetContent() throws MalformedURLException, IOException { final ResponseHeader okResponse = new ResponseHeader(HttpStatus.SC_OK); - DigestURL url = new DigestURL("http://yacy.net"); + DigestURL url = new DigestURL("https://yacy.net"); byte[] urlHash = url.hash(); byte[] fileContent = TEXT_CONTENT.getBytes(StandardCharsets.UTF_8); @@ -129,10 +129,10 @@ public void testGetContent() throws MalformedURLException, IOException { * Store an entry larger than backend buffer sizes, likely to be flushed * to the file system */ - url = new DigestURL("http://json.org"); + url = new DigestURL("https://json.org"); urlHash = url.hash(); fileContent = generateFlushableContent(); - Cache.store(new DigestURL("http://json.org"), okResponse, fileContent); + Cache.store(new DigestURL("https://json.org"), okResponse, fileContent); Cache.commit(); /* content is in cache */ assertArrayEquals(fileContent, Cache.getContent(urlHash)); @@ -141,7 +141,7 @@ public void testGetContent() throws MalformedURLException, IOException { @Test public void testDelete() throws MalformedURLException, IOException { final ResponseHeader okResponse = new ResponseHeader(HttpStatus.SC_OK); - final DigestURL url = new DigestURL("http://yacy.net"); + final DigestURL url = new DigestURL("https://yacy.net"); final byte[] urlHash = url.hash(); final byte[] fileContent = TEXT_CONTENT.getBytes(StandardCharsets.UTF_8); @@ -189,7 +189,7 @@ public void testDelete() throws MalformedURLException, IOException { @Test public void testClear() throws MalformedURLException, IOException { final ResponseHeader okResponse = new ResponseHeader(HttpStatus.SC_OK); - final DigestURL url = new DigestURL("http://yacy.net"); + final DigestURL url = new DigestURL("https://yacy.net"); final byte[] urlHash = url.hash(); byte[] fileContent = TEXT_CONTENT.getBytes(StandardCharsets.UTF_8); @@ -494,7 +494,7 @@ public static void main(final String args[]) throws IOException { for (int count = 0; count < threads; count++) { List urls = new ArrayList<>(); for (int i = 0; i < urlsPerThread; i++) { - urls.add(new DigestURL("http://yacy.net/" + i + "/" + count)); + urls.add(new DigestURL("https://yacy.net/" + i + "/" + count)); } CacheAccessTask thread = new CacheAccessTask(urls, steps, contentSize, sleepTime); thread.start(); diff --git a/test/java/net/yacy/data/wiki/WikiCodeTest.java b/test/java/net/yacy/data/wiki/WikiCodeTest.java index 1bb7c2fecd..8a1d693465 100644 --- a/test/java/net/yacy/data/wiki/WikiCodeTest.java +++ b/test/java/net/yacy/data/wiki/WikiCodeTest.java @@ -169,14 +169,14 @@ public void testExternalLink() { WikiCode wc = new WikiCode(); /* Unamed link */ - String result = wc.transform("http://wiki:8080", "[http://yacy.net]"); + String result = wc.transform("http://wiki:8080", "[https://yacy.net]"); assertTrue(result.contains("YaCy<")); /* Lua Script array parameter : should not crash the transform process */ diff --git a/test/java/net/yacy/document/parser/GenericXMLParserTest.java b/test/java/net/yacy/document/parser/GenericXMLParserTest.java index 41115b3046..9d84ac73a0 100644 --- a/test/java/net/yacy/document/parser/GenericXMLParserTest.java +++ b/test/java/net/yacy/document/parser/GenericXMLParserTest.java @@ -279,7 +279,7 @@ public void testParseAttributeURLs() throws Exception { + "" + "" + "" + "XHTML attributes URLs test" + "" + "" - + "Here are YaCyhome page and International Forum." + + "Here are YaCyhome page and International Forum." + "And this is a relative link to a sub document." + "" + ""; @@ -294,8 +294,8 @@ public void testParseAttributeURLs() throws Exception { final Collection detectedAnchors = documents[0].getAnchors(); assertNotNull(detectedAnchors); assertEquals(3, detectedAnchors.size()); - assertTrue(detectedAnchors.contains(new AnchorURL("http://www.w3.org/1999/xhtml"))); - assertTrue(detectedAnchors.contains(new AnchorURL("http://yacy.net"))); + assertTrue(detectedAnchors.contains(new AnchorURL("https://www.w3.org/1999/xhtml"))); + assertTrue(detectedAnchors.contains(new AnchorURL("https://yacy.net"))); assertTrue(detectedAnchors.contains(new AnchorURL("https://community.searchlab.eu"))); } finally { inStream.close(); @@ -315,7 +315,7 @@ public void testParseContentURLs() throws Exception { + "" + "" + "" + "XHTML content URLs test" + "" + "" + "Here are some YaCy links:" + "
    " - + "
    Home page
    " + "
    http://yacy.net
    " + "
    International Forum
    " + + "
    Home page
    " + "
    https://yacy.net
    " + "
    International Forum
    " + "
    https://community.searchlab.eu
    " + "
    " + "And this is a mention to a relative link : /document.html " + "" + ""; @@ -330,8 +330,8 @@ public void testParseContentURLs() throws Exception { final Collection detectedAnchors = documents[0].getAnchors(); assertNotNull(detectedAnchors); assertEquals(3, detectedAnchors.size()); - assertTrue(detectedAnchors.contains(new AnchorURL("http://www.w3.org/1999/xhtml"))); - assertTrue(detectedAnchors.contains(new AnchorURL("http://yacy.net"))); + assertTrue(detectedAnchors.contains(new AnchorURL("https://www.w3.org/1999/xhtml"))); + assertTrue(detectedAnchors.contains(new AnchorURL("https://yacy.net"))); assertTrue(detectedAnchors.contains(new AnchorURL("https://community.searchlab.eu"))); } finally { inStream.close(); @@ -373,7 +373,7 @@ public void testParseWithLimits() throws Exception { + "" + "" + "" + "XHTML content URLs test" + "" + "" + "

    Here are some YaCy URLs: " - + "Home page : http://yacy.net - International Forum : " + + "Home page : https://yacy.net - International Forum : " + "https://community.searchlab.eu " + "and this is a mention to a relative URL : /document.html

    " + "

    Here are YaCybug tracker and Wiki." @@ -395,10 +395,9 @@ public void testParseWithLimits() throws Exception { final Collection detectedAnchors = documents[0].getAnchors(); assertNotNull(detectedAnchors); assertEquals(5, detectedAnchors.size()); - assertTrue(detectedAnchors.contains(new AnchorURL("http://www.w3.org/1999/xhtml"))); - assertTrue(detectedAnchors.contains(new AnchorURL("http://yacy.net"))); + assertTrue(detectedAnchors.contains(new AnchorURL("https://www.w3.org/1999/xhtml"))); + assertTrue(detectedAnchors.contains(new AnchorURL("https://yacy.net"))); assertTrue(detectedAnchors.contains(new AnchorURL("https://community.searchlab.eu"))); - assertTrue(detectedAnchors.contains(new AnchorURL("http://mantis.tokeek.de"))); assertTrue(detectedAnchors.contains(new AnchorURL("https://wiki.yacy.net/index.php/"))); } finally { inStream.close(); @@ -418,8 +417,8 @@ public void testParseWithLimits() throws Exception { final Collection detectedAnchors = documents[0].getAnchors(); assertNotNull(detectedAnchors); assertEquals(2, detectedAnchors.size()); - assertTrue(detectedAnchors.contains(new AnchorURL("http://www.w3.org/1999/xhtml"))); - assertTrue(detectedAnchors.contains(new AnchorURL("http://yacy.net"))); + assertTrue(detectedAnchors.contains(new AnchorURL("https://www.w3.org/1999/xhtml"))); + assertTrue(detectedAnchors.contains(new AnchorURL("https://yacy.net"))); } finally { inStream.close(); } @@ -433,7 +432,7 @@ public void testParseWithLimits() throws Exception { .append("XHTML content URLs test") .append("") .append("

    Here are some YaCy URLs: ") - .append("Home page : http://yacy.net - International Forum : ") + .append("Home page : https://yacy.net - International Forum : ") .append("https://community.searchlab.eu ") .append("and this is a mention to a relative URL : /document.html

    "); @@ -458,8 +457,8 @@ public void testParseWithLimits() throws Exception { final Collection detectedAnchors = documents[0].getAnchors(); assertNotNull(detectedAnchors); assertEquals(3, detectedAnchors.size()); - assertTrue(detectedAnchors.contains(new AnchorURL("http://www.w3.org/1999/xhtml"))); - assertTrue(detectedAnchors.contains(new AnchorURL("http://yacy.net"))); + assertTrue(detectedAnchors.contains(new AnchorURL("https://www.w3.org/1999/xhtml"))); + assertTrue(detectedAnchors.contains(new AnchorURL("https://yacy.net"))); assertTrue(detectedAnchors.contains(new AnchorURL("https://community.searchlab.eu"))); } finally { inStream.close(); diff --git a/test/java/net/yacy/document/parser/html/ContentScraperTest.java b/test/java/net/yacy/document/parser/html/ContentScraperTest.java index cdb4f39510..92ecfb0729 100644 --- a/test/java/net/yacy/document/parser/html/ContentScraperTest.java +++ b/test/java/net/yacy/document/parser/html/ContentScraperTest.java @@ -173,7 +173,7 @@ public void testGetStartDates() throws MalformedURLException, IOException { */ @Test public void testFindAbsoluteURLs() throws MalformedURLException { - final String[] urlStrings = { "http://yacy.net", "https://community.searchlab.eu", "https://en.wikipedia.org" }; + final String[] urlStrings = { "https://yacy.net", "https://community.searchlab.eu", "https://en.wikipedia.org" }; final List urls = new ArrayList<>(); for (final String urlString : urlStrings) { urls.add(new AnchorURL(urlString)); @@ -196,39 +196,39 @@ public void testFindAbsoluteURLs() throws MalformedURLException { } /* URLs surrounded with parenthesis */ - final String[] texts = { "(http://yacy.net)", "YaCy home page (http://yacy.net)", - "Nested parentheses (YaCy home page (http://yacy.net))", - "Text in parenthesis (example : http://yacy.net)", "A markdown link [YaCy home page](http://yacy.net)", - "A markdown [example](http://yacy.net \"YaCy home page\") inline link" }; + final String[] texts = { "(https://yacy.net)", "YaCy home page (https://yacy.net)", + "Nested parentheses (YaCy home page (https://yacy.net))", + "Text in parenthesis (example : https://yacy.net)", "A markdown link [YaCy home page](https://yacy.net)", + "A markdown [example](https://yacy.net \"YaCy home page\") inline link" }; for (final String text : texts) { final Collection detectedURLs = new ArrayList<>(); ContentScraper.findAbsoluteURLs(text, detectedURLs, null); Assert.assertEquals(1, detectedURLs.size()); - Assert.assertEquals(new AnchorURL("http://yacy.net"), detectedURLs.iterator().next()); + Assert.assertEquals(new AnchorURL("https://yacy.net"), detectedURLs.iterator().next()); } /* URLs surrounded with square brackets */ //http://[abcd:ef01:2345:6789:abcd:ef01:2345:6789]/ - final String[] squareBracketsTexts = { "[http://yacy.net]", "YaCy home page [http://yacy.net]", - "Nested brackets [YaCy home page [http://yacy.net]]", - "A mediawiki external link with different label [http://yacy.net YaCy home page]" }; + final String[] squareBracketsTexts = { "[https://yacy.net]", "YaCy home page [https://yacy.net]", + "Nested brackets [YaCy home page [https://yacy.net]]", + "A mediawiki external link with different label [https://yacy.net YaCy home page]" }; for(final String text : squareBracketsTexts) { final Collection detectedURLs = new ArrayList<>(); ContentScraper.findAbsoluteURLs(text, detectedURLs, null); Assert.assertEquals(1, detectedURLs.size()); - Assert.assertEquals(new AnchorURL("http://yacy.net"), detectedURLs.iterator().next()); + Assert.assertEquals(new AnchorURL("https://yacy.net"), detectedURLs.iterator().next()); } /* URLs surrounded with curly brackets */ //http://[abcd:ef01:2345:6789:abcd:ef01:2345:6789]/ - final String[] curlyBracketsTexts = { "{http://yacy.net}", "YaCy home page {http://yacy.net}", - "Nested brackets {YaCy home page {http://yacy.net}}", - "Text in brackets {example : http://yacy.net}" }; + final String[] curlyBracketsTexts = { "{https://yacy.net}", "YaCy home page {https://yacy.net}", + "Nested brackets {YaCy home page {https://yacy.net}}", + "Text in brackets {example : https://yacy.net}" }; for(final String text : curlyBracketsTexts) { final Collection detectedURLs = new ArrayList<>(); ContentScraper.findAbsoluteURLs(text, detectedURLs, null); Assert.assertEquals(1, detectedURLs.size()); - Assert.assertEquals(new AnchorURL("http://yacy.net"), detectedURLs.iterator().next()); + Assert.assertEquals(new AnchorURL("https://yacy.net"), detectedURLs.iterator().next()); } /* URL with parenthesis */ @@ -277,7 +277,7 @@ public void testFindAbsoluteURLs() throws MalformedURLException { */ @Test public void testFindAbsoluteURLsMaxURLs() throws MalformedURLException { - final String text = "Some test URLS : http://yacy.net - https://community.searchlab.eu - https://en.wikipedia.org"; + final String text = "Some test URLS : https://yacy.net - https://community.searchlab.eu - https://en.wikipedia.org"; /* No limit */ ArrayList detectedURLs = new ArrayList<>(); diff --git a/test/java/net/yacy/document/parser/html/IconEntryTest.java b/test/java/net/yacy/document/parser/html/IconEntryTest.java index ee4317735b..3461cee758 100644 --- a/test/java/net/yacy/document/parser/html/IconEntryTest.java +++ b/test/java/net/yacy/document/parser/html/IconEntryTest.java @@ -81,7 +81,7 @@ public final void testGetClosestSize() throws MalformedURLException { sizes.add(new Dimension(16,16)); Dimension preferredSize = new Dimension(16, 16); - IconEntry icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes); + IconEntry icon = new IconEntry(new DigestURL("https://yacy.net"), rels, sizes); Dimension result = icon.getClosestSize(preferredSize); Assert.assertEquals(preferredSize, result); @@ -103,14 +103,14 @@ public final void testGetClosestSize() throws MalformedURLException { /* Sizes set contains only one item */ sizes = new HashSet<>(); sizes.add(new Dimension(128,128)); - icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes); + icon = new IconEntry(new DigestURL("https://yacy.net"), rels, sizes); preferredSize = new Dimension(1992, 1024); result = icon.getClosestSize(preferredSize); Assert.assertEquals(new Dimension(128, 128), result); /* Empty sizes set */ sizes = new HashSet<>(); - icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes); + icon = new IconEntry(new DigestURL("https://yacy.net"), rels, sizes); preferredSize = new Dimension(16, 16); result = icon.getClosestSize(preferredSize); Assert.assertNull(result); @@ -120,7 +120,7 @@ public final void testGetClosestSize() throws MalformedURLException { sizes.add(new Dimension(128,128)); sizes.add(new Dimension(256,512)); sizes.add(new Dimension(16,16)); - icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes); + icon = new IconEntry(new DigestURL("https://yacy.net"), rels, sizes); preferredSize = null; result = icon.getClosestSize(preferredSize); Assert.assertNull(result); @@ -137,7 +137,7 @@ public final void testSizesToString() throws MalformedURLException { sizes.add(new Dimension(256,512)); sizes.add(new Dimension(16,16)); - IconEntry icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes); + IconEntry icon = new IconEntry(new DigestURL("https://yacy.net"), rels, sizes); String sizesStr = icon.sizesToString(); /* The set is not ordered, only check result contains what we expect */ Assert.assertTrue(sizesStr.contains("128x128")); @@ -149,14 +149,14 @@ public final void testSizesToString() throws MalformedURLException { sizes = new HashSet<>(); sizes.add(new Dimension(128,128)); - icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes); + icon = new IconEntry(new DigestURL("https://yacy.net"), rels, sizes); sizesStr = icon.sizesToString(); Assert.assertEquals("128x128", sizesStr); /* Empty sizes set */ sizes = new HashSet<>(); - icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes); + icon = new IconEntry(new DigestURL("https://yacy.net"), rels, sizes); sizesStr = icon.sizesToString(); Assert.assertTrue(sizesStr.isEmpty()); } @@ -172,7 +172,7 @@ public final void testRelToString() throws MalformedURLException { Set sizes = new HashSet<>(); sizes.add(new Dimension(128,128)); - IconEntry icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes); + IconEntry icon = new IconEntry(new DigestURL("https://yacy.net"), rels, sizes); String relStr = icon.relToString(); /* The set is not ordered, only check result contains what we expect */ Assert.assertTrue(relStr.contains(IconLinkRelations.ICON.getRelValue())); @@ -184,7 +184,7 @@ public final void testRelToString() throws MalformedURLException { rels = new HashSet<>(); rels.add(IconLinkRelations.ICON.getRelValue()); - icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes); + icon = new IconEntry(new DigestURL("https://yacy.net"), rels, sizes); relStr = icon.relToString(); Assert.assertEquals(IconLinkRelations.ICON.getRelValue(), relStr); } diff --git a/test/java/net/yacy/document/parser/ooxmlParserTest.java b/test/java/net/yacy/document/parser/ooxmlParserTest.java index 12d30dee27..9d3c2a6e99 100644 --- a/test/java/net/yacy/document/parser/ooxmlParserTest.java +++ b/test/java/net/yacy/document/parser/ooxmlParserTest.java @@ -152,7 +152,7 @@ public void testParseURLs() throws Exception { final Collection anchors = docs[0].getAnchors(); assertNotNull("Detected URLs must not be null", anchors); assertEquals("2 URLs should be detected", 2, anchors.size()); - assertTrue("YaCy home page URL should have been parsed: " + anchors.toString(), anchors.contains(new AnchorURL("http://yacy.net/"))); + assertTrue("YaCy home page URL should have been parsed: " + anchors.toString(), anchors.contains(new AnchorURL("https://yacy.net/"))); assertTrue("YaCy forum URL should have been parsed: " + anchors.toString(), anchors.contains(new AnchorURL("https://community.searchlab.eu/"))); } } diff --git a/test/net/yacy/data/URLLicenseConcurrentTest.java b/test/net/yacy/data/URLLicenseConcurrentTest.java index 9bc49e821d..1f1c3e9c3e 100644 --- a/test/net/yacy/data/URLLicenseConcurrentTest.java +++ b/test/net/yacy/data/URLLicenseConcurrentTest.java @@ -41,7 +41,7 @@ public class URLLicenseConcurrentTest { */ private static class ClientThread extends Thread { - private String testURL = "http://yacy.net"; + private String testURL = "https://yacy.net"; private int steps = 100000;