From bfd017a035aec8e45ed2309f72b878431185b975 Mon Sep 17 00:00:00 2001 From: wouter Date: Fri, 12 Dec 2025 16:35:42 +0100 Subject: [PATCH 1/3] Prioritize Swedish name API about CoL --- server.js | 70 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/server.js b/server.js index 185f8b2..c3a3ed6 100644 --- a/server.js +++ b/server.js @@ -885,7 +885,42 @@ let getName = async (sciNameId, sciName, force = false, country = null) => { // If all languages are already filled, skip further fetching if (missingLanguages.length !== 0) { - // Priority 1: GBIF (Catalog of Life) + + // Priority 1: Artdatabanken.se (Swedish) + if (!nameResult.vernacularNames.sv) { + try { + const swedishUrl = encodeURI(`https://nos-api.artdatabanken.se/api/search?searchType=exact&search=${nameResult.scientificName}`); + const swedishResponse = await axios + .get(swedishUrl, { + timeout: 3000, + headers: { + 'Accept-Encoding': 'gzip', + 'User-Agent': 'Artsorakel backend bot/4.0 (https://www.artsdatabanken.no) axios/0.21.1' + } + }) + .catch((error) => { + console.log(`Failed to get Swedish name for ${nameResult.scientificName}:`, error.message); + return null; + }); + + if (swedishResponse && swedishResponse.data && Array.isArray(swedishResponse.data)) { + const matchingTaxon = swedishResponse.data.find( + item => item.scientificName && + item.scientificName.toLowerCase() === nameResult.scientificName.toLowerCase() && + item.swedishName + ); + + if (matchingTaxon && matchingTaxon.swedishName) { + nameResult.vernacularNames.sv = matchingTaxon.swedishName; + } + } + } catch (error) { + console.log(`Error fetching Swedish name for ${nameResult.scientificName}:`, error.message); + } + } + + + // Priority 2: GBIF (Catalog of Life) if (missingLanguages.length > 0) { try { const gbifUrl = encodeURI(`https://api.gbif.org/v1/species/search?datasetKey=7ddf754f-d193-4cc9-b351-99906754a03b&nameType=SCIENTIFIC&q=${nameResult.scientificName}`); @@ -936,39 +971,6 @@ let getName = async (sciNameId, sciName, force = false, country = null) => { } } - // Priority 2: Artdatabanken.se (Swedish) - if (!nameResult.vernacularNames.sv) { - try { - const swedishUrl = encodeURI(`https://nos-api.artdatabanken.se/api/search?searchType=exact&search=${nameResult.scientificName}`); - const swedishResponse = await axios - .get(swedishUrl, { - timeout: 3000, - headers: { - 'Accept-Encoding': 'gzip', - 'User-Agent': 'Artsorakel backend bot/4.0 (https://www.artsdatabanken.no) axios/0.21.1' - } - }) - .catch((error) => { - console.log(`Failed to get Swedish name for ${nameResult.scientificName}:`, error.message); - return null; - }); - - if (swedishResponse && swedishResponse.data && Array.isArray(swedishResponse.data)) { - const matchingTaxon = swedishResponse.data.find( - item => item.scientificName && - item.scientificName.toLowerCase() === nameResult.scientificName.toLowerCase() && - item.swedishName - ); - - if (matchingTaxon && matchingTaxon.swedishName) { - nameResult.vernacularNames.sv = matchingTaxon.swedishName; - } - } - } catch (error) { - console.log(`Error fetching Swedish name for ${nameResult.scientificName}:`, error.message); - } - } - // Priority 3: Wikipedia const remainingLangs = targetLanguages.filter(lang => !nameResult.vernacularNames[lang]); if (remainingLangs.length > 0) { From 2143beae29bf6ec149610135cfcab8627c99a18c Mon Sep 17 00:00:00 2001 From: wouter Date: Fri, 12 Dec 2025 16:47:32 +0100 Subject: [PATCH 2/3] Update from main --- server.js | 292 +++++++++++++++++++++++++++++------------------------- 1 file changed, 155 insertions(+), 137 deletions(-) diff --git a/server.js b/server.js index c3a3ed6..e60cf5d 100644 --- a/server.js +++ b/server.js @@ -372,7 +372,7 @@ const authenticateApiToken = (req, res, next) => { return next(); } - if (validTokens[token] && validTokens[token].enabled === true) { + if (validTokens?.[token]?.enabled === true) { req.auth = { type: 'api', token: token, @@ -466,6 +466,7 @@ const storage = multer.memoryStorage(); const upload = multer({ storage: storage }); let getPicture = (sciName) => { + if (!sciName) return null; sciName = sciName.replaceAll("×", "x").replaceAll("ë", "e"); let pic = taxonPics[sciName]; @@ -479,7 +480,7 @@ let getPicture = (sciName) => { let writelog = (req, json, auth = null) => { let application; - if (auth && auth.application) { + if (auth?.application) { application = sanitize(auth.application); } else if (req.body.application) { application = sanitize(req.body.application); @@ -538,10 +539,10 @@ let getName = async (sciNameId, sciName, force = false, country = null) => { try { const cachedData = JSON.parse(fs.readFileSync(existingFile)); if (country === 'NO') { - if (cachedData.redListCategories && cachedData.redListCategories.NO) { + if (cachedData?.redListCategories?.NO) { cachedData.redListCategory = cachedData.redListCategories.NO; } - if (cachedData.invasiveCategories && cachedData.invasiveCategories.NO) { + if (cachedData?.invasiveCategories?.NO) { cachedData.invasiveCategory = cachedData.invasiveCategories.NO; } } @@ -585,10 +586,10 @@ let getName = async (sciNameId, sciName, force = false, country = null) => { try { const cachedData = JSON.parse(fs.readFileSync(jsonfilename)); if (country === 'NO') { - if (cachedData.redListCategories && cachedData.redListCategories.NO) { + if (cachedData?.redListCategories?.NO) { cachedData.redListCategory = cachedData.redListCategories.NO; } - if (cachedData.invasiveCategories && cachedData.invasiveCategories.NO) { + if (cachedData?.invasiveCategories?.NO) { cachedData.invasiveCategory = cachedData.invasiveCategories.NO; } } @@ -698,7 +699,7 @@ let getName = async (sciNameId, sciName, force = false, country = null) => { else { try { let url = encodeURI( - `https://artsdatabanken.no/api/Resource/?Take=10&Type=taxon&Name=${sciName}` + `https://artsdatabanken.no/api/Resource/?Take=250&Type=taxon&Name=${sciName}` ); let taxon = await axios .get(url, { @@ -732,19 +733,56 @@ let getName = async (sciNameId, sciName, force = false, country = null) => { return nameResult; } - let sciNameIdFromName = resourceObject.AcceptedNameUsage.ScientificNameId + if (!!resourceObject) { + let sciNameIdFromName = resourceObject.AcceptedNameUsage.ScientificNameId + + try { + let url = encodeURI( + `https://artsdatabanken.no/Api/Taxon/ScientificName/${sciNameIdFromName}` + ); + scientificNameIdObject = await axios + .get(url, { + timeout: 3000, + headers: { + 'Accept-Encoding': 'gzip', + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:145.0) Gecko/20100101 Firefox/145.0' + } + }) + .catch((error) => { + writeErrorLog( + `Failed to ${!force ? "get info for" : "*recache*" + } ${sciName} from ${url}.`, + error + ); + throw ""; + }); + scientificNameIdObject = scientificNameIdObject.data + } + catch (error) { + writeErrorLog( + `Error in getName(${sciNameId}) for scientificNameIdObject from name. Retry: ${encodeURI( + server_url + "/cachetaxon/id/" + sciNameId + )}.`, + error + ); + return nameResult; + } + } + } + if (!!resourceObject) { + try { let url = encodeURI( - `https://artsdatabanken.no/Api/Taxon/ScientificName/${sciNameIdFromName}` + `https://lister.artsdatabanken.no/odata/v1/alienspeciesassessment${listVersions.AlienSpeciesList}?filter=scientificName/ScientificNameId eq ${resourceObject.AcceptedNameUsage.ScientificNameId}&select=category` ); - scientificNameIdObject = await axios + alienSpeciesListObject = await axios .get(url, { timeout: 3000, headers: { 'Accept-Encoding': 'gzip', - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:145.0) Gecko/20100101 Firefox/145.0' + 'User-Agent': 'Artsorakel backend bot/4.0 (https://www.artsdatabanken.no) axios/0.21.1' } }) .catch((error) => { @@ -755,129 +793,99 @@ let getName = async (sciNameId, sciName, force = false, country = null) => { ); throw ""; }); - scientificNameIdObject = scientificNameIdObject.data + alienSpeciesListObject = alienSpeciesListObject.data } catch (error) { writeErrorLog( - `Error in getName(${sciNameId}) for scientificNameIdObject from name. Retry: ${encodeURI( + `Error in getName(${sciNameId}) for alienSpeciesListObject. Retry: ${encodeURI( server_url + "/cachetaxon/id/" + sciNameId )}.`, error ); return nameResult; } - } + try { + let url = encodeURI( + `https://lister.artsdatabanken.no/odata/v1/speciesassessment${listVersions.Redlist}?filter=ScientificNameId eq ${resourceObject.AcceptedNameUsage.ScientificNameId}&select=category` + ); + redListObject = await axios + .get(url, { + timeout: 3000, + headers: { + 'Accept-Encoding': 'gzip', + 'User-Agent': 'Artsorakel backend bot/4.0 (https://www.artsdatabanken.no) axios/0.21.1' + } + }) + .catch((error) => { + writeErrorLog( + `Failed to ${!force ? "get info for" : "*recache*" + } ${sciName} from ${url}.`, + error + ); + throw ""; + }); + redListObject = redListObject.data + } + catch (error) { + writeErrorLog( + `Error in getName(${sciNameId}) for redListObject. Retry: ${encodeURI( + server_url + "/cachetaxon/id/" + sciNameId + )}.`, + error + ); + return nameResult; + } - try { - let url = encodeURI( - `https://lister.artsdatabanken.no/odata/v1/alienspeciesassessment${listVersions.AlienSpeciesList}?filter=scientificName/ScientificNameId eq ${resourceObject.AcceptedNameUsage.ScientificNameId}&select=category` - ); - alienSpeciesListObject = await axios - .get(url, { - timeout: 3000, - headers: { - 'Accept-Encoding': 'gzip', - 'User-Agent': 'Artsorakel backend bot/4.0 (https://www.artsdatabanken.no) axios/0.21.1' - } - }) - .catch((error) => { - writeErrorLog( - `Failed to ${!force ? "get info for" : "*recache*" - } ${sciName} from ${url}.`, - error - ); - throw ""; - }); - alienSpeciesListObject = alienSpeciesListObject.data - } - catch (error) { - writeErrorLog( - `Error in getName(${sciNameId}) for alienSpeciesListObject. Retry: ${encodeURI( - server_url + "/cachetaxon/id/" + sciNameId - )}.`, - error - ); - return nameResult; - } - - - try { - let url = encodeURI( - `https://lister.artsdatabanken.no/odata/v1/speciesassessment${listVersions.Redlist}?filter=ScientificNameId eq ${resourceObject.AcceptedNameUsage.ScientificNameId}&select=category` - ); - redListObject = await axios - .get(url, { - timeout: 3000, - headers: { - 'Accept-Encoding': 'gzip', - 'User-Agent': 'Artsorakel backend bot/4.0 (https://www.artsdatabanken.no) axios/0.21.1' - } - }) - .catch((error) => { - writeErrorLog( - `Failed to ${!force ? "get info for" : "*recache*" - } ${sciName} from ${url}.`, - error - ); - throw ""; - }); - redListObject = redListObject.data - } - catch (error) { - writeErrorLog( - `Error in getName(${sciNameId}) for redListObject. Retry: ${encodeURI( - server_url + "/cachetaxon/id/" + sciNameId - )}.`, - error - ); - return nameResult; - } + // We should now have the objects we need to get and store the json + if (resourceObject.AcceptedNameUsage?.ScientificName) { + nameResult.scientificName = resourceObject.AcceptedNameUsage.ScientificName + } + if (redListObject?.value?.length) { + nameResult.redListCategories.NO = redListObject.value[0].category; + } - // We should now have the objects we need to get and store the json - nameResult.scientificName = resourceObject.AcceptedNameUsage.ScientificName + if (alienSpeciesListObject?.value?.length) { + nameResult.invasiveCategories.NO = alienSpeciesListObject.value[0].category; + } - if (redListObject.value.length) { - nameResult.redListCategories.NO = redListObject.value[0].category; - } + if (scientificNameIdObject?.dynamicProperties) { + let artsobsname = scientificNameIdObject.dynamicProperties.find( + (dp) => + dp.Name === "GruppeNavn" && + dp.Properties.find((p) => p.Value === "Artsobservasjoner") + ); - if (alienSpeciesListObject.value.length) { - nameResult.invasiveCategories.NO = alienSpeciesListObject.value[0].category; - } + if (artsobsname?.Value?.trim()) { + const rawGroupName = artsobsname.Value.toLowerCase(); + const capitalizedGroupName = capitalizeFirstLetter(artsobsname.Value); - if (scientificNameIdObject.dynamicProperties) { - let artsobsname = scientificNameIdObject.dynamicProperties.find( - (dp) => - dp.Name === "GruppeNavn" && - dp.Properties.find((p) => p.Value === "Artsobservasjoner") - ); + if (groupNameTranslations[rawGroupName]) { + nameResult.groupNames = groupNameTranslations[rawGroupName]; + nameResult.groupName = capitalizedGroupName; + } else { + nameResult.groupName = capitalizedGroupName; + nameResult.groupNames = { 'nb': capitalizedGroupName, 'nn': capitalizedGroupName, 'se': capitalizedGroupName }; + } + } + } - if (artsobsname && artsobsname.Value && artsobsname.Value.trim()) { - const rawGroupName = artsobsname.Value.toLowerCase(); - const capitalizedGroupName = capitalizeFirstLetter(artsobsname.Value); - if (groupNameTranslations[rawGroupName]) { - nameResult.groupNames = groupNameTranslations[rawGroupName]; - nameResult.groupName = capitalizedGroupName; - } else { - nameResult.groupName = capitalizedGroupName; - nameResult.groupNames = { 'nb': capitalizedGroupName, 'nn': capitalizedGroupName, 'se': capitalizedGroupName }; + // Extract all RecommendedVernacularName fields for different languages from Artsdatabanken + for (const [key, value] of Object.entries(resourceObject)) { + if (key.startsWith("RecommendedVernacularName_") && value) { + let langCode = key.replace("RecommendedVernacularName_", ""); + langCode = langCode.split("-")[0] + nameResult.vernacularNames[langCode] = value; } } + } - // Extract all RecommendedVernacularName fields for different languages from Artsdatabanken - for (const [key, value] of Object.entries(resourceObject)) { - if (key.startsWith("RecommendedVernacularName_") && value) { - let langCode = key.replace("RecommendedVernacularName_", ""); - langCode = langCode.split("-")[0] - nameResult.vernacularNames[langCode] = value; - } - } // Define target languages to fetch (excluding those typically found in Artsdatabanken) const targetLanguages = ['sv', 'nl', 'en', 'es']; @@ -903,14 +911,14 @@ let getName = async (sciNameId, sciName, force = false, country = null) => { return null; }); - if (swedishResponse && swedishResponse.data && Array.isArray(swedishResponse.data)) { + if (swedishResponse?.data && Array.isArray(swedishResponse.data)) { const matchingTaxon = swedishResponse.data.find( item => item.scientificName && item.scientificName.toLowerCase() === nameResult.scientificName.toLowerCase() && item.swedishName ); - if (matchingTaxon && matchingTaxon.swedishName) { + if (matchingTaxon?.swedishName) { nameResult.vernacularNames.sv = matchingTaxon.swedishName; } } @@ -937,7 +945,7 @@ let getName = async (sciNameId, sciName, force = false, country = null) => { return null; }); - if (gbifResponse && gbifResponse.data && gbifResponse.data.results && Array.isArray(gbifResponse.data.results)) { + if (gbifResponse?.data?.results && Array.isArray(gbifResponse.data.results)) { const matchingResults = gbifResponse.data.results.filter( item => item.canonicalName && item.canonicalName.toLowerCase() === nameResult.scientificName.toLowerCase() && @@ -958,7 +966,7 @@ let getName = async (sciNameId, sciName, force = false, country = null) => { const nameEntry = result.vernacularNames.find( vn => vn.language === threeLetterCode && vn.vernacularName ); - if (nameEntry && nameEntry.vernacularName) { + if (nameEntry?.vernacularName) { nameResult.vernacularNames[twoLetterCode] = nameEntry.vernacularName; } } @@ -971,6 +979,8 @@ let getName = async (sciNameId, sciName, force = false, country = null) => { } } + + // Priority 3: Wikipedia const remainingLangs = targetLanguages.filter(lang => !nameResult.vernacularNames[lang]); if (remainingLangs.length > 0) { @@ -989,7 +999,7 @@ let getName = async (sciNameId, sciName, force = false, country = null) => { return null; }); - if (wikiResponse && wikiResponse.data && Array.isArray(wikiResponse.data)) { + if (wikiResponse?.data && Array.isArray(wikiResponse.data)) { for (const link of wikiResponse.data) { if (remainingLangs.includes(link.code) && !nameResult.vernacularNames[link.code]) { // Remove parentheses and their contents from the title @@ -1024,14 +1034,14 @@ let getName = async (sciNameId, sciName, force = false, country = null) => { return null; }); - if (iNatResponse && iNatResponse.data && iNatResponse.data.results && Array.isArray(iNatResponse.data.results)) { + if (iNatResponse?.data?.results && Array.isArray(iNatResponse.data.results)) { const result = iNatResponse.data.results.find( item => item.name && item.name.toLowerCase() === nameResult.scientificName.toLowerCase() && item.preferred_common_name ); - if (result && result.preferred_common_name) { + if (result?.preferred_common_name) { nameResult.vernacularNames[lang] = result.preferred_common_name; } } @@ -1049,24 +1059,29 @@ let getName = async (sciNameId, sciName, force = false, country = null) => { nameResult.scientificName || sciName; - if (resourceObject.Description) { - const description = - resourceObject.Description.find( - (desc) => - desc.Language == "nb" || - desc.Language == "no" || - desc.Language == "nn" - ) || resourceObject.Description[0]; - - nameResult.infoUrl = description.Id.replace( - "Nodes/", - "https://artsdatabanken.no/Pages/" - ); - } else { - nameResult.infoUrl = - "https://artsdatabanken.no/" + resourceObject.Id; + + if (!!resourceObject) { + if (resourceObject.Description) { + const description = + resourceObject.Description.find( + (desc) => + desc.Language == "nb" || + desc.Language == "no" || + desc.Language == "nn" + ) || resourceObject.Description[0]; + + nameResult.infoUrl = description.Id.replace( + "Nodes/", + "https://artsdatabanken.no/Pages/" + ); + } else { + nameResult.infoUrl = + "https://artsdatabanken.no/" + resourceObject.Id; + } + } + let jsonfilename; if (sciNameId) { const filename = `${sciNameId}_${nameResult.scientificName}`; @@ -1618,14 +1633,14 @@ let getId = async (req) => { if (Object.keys(nameResult.redListCategories).length > 0) { pred.redListCategories = nameResult.redListCategories - if (!!nameResult.redListCategories && nameResult.redListCategories.NO) { + if (nameResult?.redListCategories?.NO) { pred.redListCategory = nameResult.redListCategories.NO } } if (Object.keys(nameResult.invasiveCategories).length > 0) { pred.invasiveCategories = nameResult.invasiveCategories - if (!!nameResult.invasiveCategories && nameResult.invasiveCategories.NO) { + if (nameResult?.invasiveCategories?.NO) { pred.invasiveCategory = nameResult.invasiveCategories.NO } } @@ -1699,7 +1714,7 @@ app.post("/identify", idLimiter, authenticateApiToken, upload.array("image"), as res.status(200).json(json); // --- Now that the reply has been sent, let each returned name have a 5% chance to be recached if its file is older than 10 days - if (json.predictions && json.predictions[0] && json.predictions[0].taxa) { + if (json?.predictions?.[0]?.taxa) { json.predictions[0].taxa.items.forEach((taxon) => { if (Math.random() < 0.05) { let filename = `${taxadir}/${encodeURIComponent( @@ -2083,7 +2098,10 @@ app.get("/", apiLimiter, (req, res) => { if (!!branch) { const gitfile = ".git/FETCH_HEAD"; if (fs.existsSync(gitfile)) { - v = (fs.readFileSync(gitfile).toString().split("\n").find(x => x.includes(branch))).split("\t")[0] + v = (fs.readFileSync(gitfile).toString().split("\n").find(x => x.includes(branch))) + if (!!v) { + v = v.split("\t")[0] + } } } @@ -2095,7 +2113,7 @@ app.get("/", apiLimiter, (req, res) => { }); -app.get("/image/*", apiLimiter, authenticateAdminToken, async (req, res) => { +app.get("/image/*", apiLimiter, async (req, res) => { const urlParam = req.originalUrl.replace("/image/", ""); const password = urlParam.split("&")[1].toString(); const id = urlParam.split("&")[0]; From 8d0d9ae0b69fcb1f11e051570f890d0439184afd Mon Sep 17 00:00:00 2001 From: Wouter Koch Date: Fri, 12 Dec 2025 17:04:32 +0100 Subject: [PATCH 3/3] Potential fix for code scanning alert no. 35: Use of externally-controlled format string Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.js b/server.js index 708f0d1..59c1a1a 100644 --- a/server.js +++ b/server.js @@ -923,7 +923,7 @@ let getName = async (sciNameId, sciName, force = false, country = null) => { } } } catch (error) { - console.log(`Error fetching Swedish name for ${nameResult.scientificName}:`, error.message); + console.log("Error fetching Swedish name for %s:", nameResult.scientificName, error.message); } }