From c3ea301911e374ba76b1146c53116d7da657419e Mon Sep 17 00:00:00 2001 From: junaid Date: Mon, 15 Jun 2026 17:49:17 +0530 Subject: [PATCH 1/2] Auto warm cache --- mbsssss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mbsssss b/mbsssss index 8751dff..a8f2a72 160000 --- a/mbsssss +++ b/mbsssss @@ -1 +1 @@ -Subproject commit 8751dff89b06bea58b57edb3593b91b277db21b8 +Subproject commit a8f2a72d734432a781a5bc9d3c0df99eff6210a1 From c172f1540a00249379e08774a38655dfc5a7a8ab Mon Sep 17 00:00:00 2001 From: junaid Date: Mon, 29 Jun 2026 16:12:15 +0530 Subject: [PATCH 2/2] Add a new XML writer to use flat fields --- .../search/solrwriter/MBFlatXMLWriter.java | 87 ++++++++ .../search/solrwriter/MBXMLWriter.java | 4 +- .../search/solrwriter/MMDObjectBuilder.java | 7 + .../entityBuilders/ArtistBuilder.java | 189 ++++++++++++++++++ mbsssss | 2 +- 5 files changed, 286 insertions(+), 3 deletions(-) create mode 100644 mb-solr/src/main/java/org/musicbrainz/search/solrwriter/MBFlatXMLWriter.java create mode 100644 mb-solr/src/main/java/org/musicbrainz/search/solrwriter/MMDObjectBuilder.java create mode 100644 mb-solr/src/main/java/org/musicbrainz/search/solrwriter/entityBuilders/ArtistBuilder.java diff --git a/mb-solr/src/main/java/org/musicbrainz/search/solrwriter/MBFlatXMLWriter.java b/mb-solr/src/main/java/org/musicbrainz/search/solrwriter/MBFlatXMLWriter.java new file mode 100644 index 0000000..ba908c2 --- /dev/null +++ b/mb-solr/src/main/java/org/musicbrainz/search/solrwriter/MBFlatXMLWriter.java @@ -0,0 +1,87 @@ +package org.musicbrainz.search.solrwriter; + +import org.apache.lucene.index.IndexableField; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.ResultContext; +import org.apache.solr.search.DocList; +import org.musicbrainz.search.solrwriter.entityBuilders.ArtistBuilder; + +import java.util.*; + +public class MBFlatXMLWriter extends MBXMLWriter { + + private MMDObjectBuilder flatBuilder; + + @Override + public void init(NamedList initArgs) { + super.init(initArgs); + flatBuilder = new ArtistBuilder(); + } + + @Override + public void parseSolrResponse(ResultContext con, MetadataListWrapper metadatalistwrapper, SolrQueryRequest req) { + DocList docList = con.getDocList(); + metadatalistwrapper.setCountAndOffset(docList.matches(), docList.offset()); + var liveList = metadatalistwrapper.getLiveList(); + float maxScore = docList.maxScore(); + Iterator iter = con.getProcessedDocuments(); + while (iter.hasNext()) { + SolrDocument solrDoc = processSolrDocument(iter.next()); + Object artist = flatBuilder.build(solrDoc); + try{ + adjustScore(maxScore, artist, Float.parseFloat(solrDoc.get("score").toString())); + } + catch(NullPointerException e) { + throw new RuntimeException(SCORE_NOT_IN_FIELD_LIST); + } + liveList.add(artist); + } + } + + @Override + public void parseSolrResponse(SolrDocumentList docList, MetadataListWrapper metadatalistwrapper) { + metadatalistwrapper.setCountAndOffset(docList.getNumFound(), docList.getStart()); + + List liveList = metadatalistwrapper.getLiveList(); + float maxScore = docList.getMaxScore(); + + for (SolrDocument doc : docList) { + Object artist = flatBuilder.build(doc); + + try { + adjustScore(maxScore, artist, Float.parseFloat(doc.get("score").toString())); + } catch (NullPointerException e) { + throw new RuntimeException(SCORE_NOT_IN_FIELD_LIST); + } + + liveList.add(artist); + } + } + + private SolrDocument processSolrDocument(SolrDocument rawDoc){ + SolrDocument cleanDoc = new SolrDocument(); + for (String fieldName : rawDoc.getFieldNames()) { + Collection values = rawDoc.getFieldValues(fieldName); + if (values == null) { + continue; + } + List cleanValues = new ArrayList<>(); + for (Object val : values) { + if (val instanceof IndexableField f) { + cleanValues.add(f.stringValue() != null ? f.stringValue() : f.numericValue()); + } else { + cleanValues.add(val); + } + } + if (cleanValues.size() == 1) { + cleanDoc.setField(fieldName, cleanValues.get(0)); + } else { + cleanDoc.setField(fieldName, cleanValues); + } + } + return cleanDoc; + } +} \ No newline at end of file diff --git a/mb-solr/src/main/java/org/musicbrainz/search/solrwriter/MBXMLWriter.java b/mb-solr/src/main/java/org/musicbrainz/search/solrwriter/MBXMLWriter.java index 1f43b57..a039584 100644 --- a/mb-solr/src/main/java/org/musicbrainz/search/solrwriter/MBXMLWriter.java +++ b/mb-solr/src/main/java/org/musicbrainz/search/solrwriter/MBXMLWriter.java @@ -106,7 +106,7 @@ public static Optional getType(String entityType) { * for different entity types, as well as calling the appropriate method to * set the list on the {@link Metadata} object. */ - class MetadataListWrapper { + public class MetadataListWrapper { private List objList = null; private Object MMDList = null; private Metadata metadata = null; @@ -311,7 +311,7 @@ public void init(NamedList initArgs) { .orElseThrow(() -> new RuntimeException("no valid entitytype given")); } - private static void adjustScore(float maxScore, Object object, + public static void adjustScore(float maxScore, Object object, float objectScore) { Method setScoreMethod; try { diff --git a/mb-solr/src/main/java/org/musicbrainz/search/solrwriter/MMDObjectBuilder.java b/mb-solr/src/main/java/org/musicbrainz/search/solrwriter/MMDObjectBuilder.java new file mode 100644 index 0000000..56db28a --- /dev/null +++ b/mb-solr/src/main/java/org/musicbrainz/search/solrwriter/MMDObjectBuilder.java @@ -0,0 +1,7 @@ +package org.musicbrainz.search.solrwriter; + +import org.apache.solr.common.SolrDocument; + +public interface MMDObjectBuilder{ + Object build(SolrDocument doc); +} \ No newline at end of file diff --git a/mb-solr/src/main/java/org/musicbrainz/search/solrwriter/entityBuilders/ArtistBuilder.java b/mb-solr/src/main/java/org/musicbrainz/search/solrwriter/entityBuilders/ArtistBuilder.java new file mode 100644 index 0000000..ef98095 --- /dev/null +++ b/mb-solr/src/main/java/org/musicbrainz/search/solrwriter/entityBuilders/ArtistBuilder.java @@ -0,0 +1,189 @@ +package org.musicbrainz.search.solrwriter.entityBuilders; + +import org.apache.solr.common.SolrDocument; +import org.musicbrainz.mmd2.*; +import org.musicbrainz.search.solrwriter.MMDObjectBuilder; + +import java.math.BigInteger; +import java.util.Objects; +import java.util.Collection; +import java.util.List; +import java.util.ArrayList; + + +public class ArtistBuilder implements MMDObjectBuilder { + @Override + public Object build(SolrDocument doc) { + Artist artist = new Artist(); + artist.setId(Objects.toString(doc.get("mbid"), null)); + artist.setName(Objects.toString(doc.get("artist"), null)); + artist.setSortName(Objects.toString(doc.get("sortname"), null)); + artist.setDisambiguation(Objects.toString(doc.get("comment"), null)); + artist.setCountry(Objects.toString(doc.get("country"), null)); + artist.setType(Objects.toString(doc.get("type_name"), null)); + artist.setTypeId(Objects.toString(doc.get("type_gid"), null)); + + artist.setGender(buildGender(doc)); + artist.setLifeSpan(buildLifeSpan(doc.get("begin"), doc.get("end"), doc.get("ended"))); + artist.setArea(buildArea(doc)); + artist.setBeginArea(buildBeginArea(doc)); + artist.setEndArea(buildEndArea(doc)); + artist.setTagList(buildTagList(doc)); + artist.setIpiList(buildIpiList(doc)); + artist.setIsniList(buildIsniList(doc)); + artist.setAliasList(buildAliasList(doc)); + + return artist; + } + + private Gender buildGender(SolrDocument doc) { + Gender gender = new Gender(); + gender.setId(Objects.toString(doc.get("gender_gid"), null)); + gender.setContent(Objects.toString(doc.get("gender_name"), null)); + return gender; + } + + private LifeSpan buildLifeSpan(Object begin, Object end, Object ended) { + LifeSpan lifeSpan = new LifeSpan(); + lifeSpan.setBegin(Objects.toString(begin, null)); + lifeSpan.setEnd(Objects.toString(end, null)); + lifeSpan.setEnded(Objects.toString(ended, null)); + return lifeSpan; + } + + private AliasList buildAliasList(SolrDocument doc) { + AliasList aliasList = new AliasList(); + List names = toStringList(doc.getFieldValues("alias")); + List primaries = toStringList(doc.getFieldValues("alias_primary_for_locale")); + List sortNames = toStringList(doc.getFieldValues("alias_sortname")); + List locales = toStringList(doc.getFieldValues("alias_locale")); + List beginDates = toStringList(doc.getFieldValues("alias_begindate")); + List endDates = toStringList(doc.getFieldValues("alias_enddate")); + List typeNames = toStringList(doc.getFieldValues("alias_type_name")); + List typeIds = toStringList(doc.getFieldValues("alias_type_gid")); + for (int i = 0; i < names.size(); i++){ + aliasList.getAlias().add(buildAlias( + names.get(i), + getOrNull(sortNames, i), + getOrNull(locales, i), + getOrNull(primaries, i), + getOrNull(beginDates, i), + getOrNull(endDates, i), + getOrNull(typeNames, i), + getOrNull(typeIds, i) + )); + } + return aliasList; + } + private static String getOrNull(List list, int index) { + return index < list.size() ? list.get(index) : null; + } + + private Alias buildAlias(String content, String sortName, String locale, String primary, + String beginDate, String endDate, String typeName, String typeId) { + Alias alias = new Alias(); + if (content != null && !content.isEmpty()) alias.setContent(content); + if (sortName != null && !sortName.isEmpty()) alias.setSortName(sortName); + if (locale != null && !locale.isEmpty()) alias.setLocale(locale); + if (primary != null && !primary.isEmpty()) alias.setPrimary(primary); + if (beginDate != null && !beginDate.isEmpty()) alias.setBeginDate(beginDate); + if (endDate != null && !endDate.isEmpty()) alias.setEndDate(endDate); + if (typeName != null && !typeName.isEmpty()) alias.setType(typeName); + if (typeId != null && !typeId.isEmpty()) alias.setTypeId(typeId); + + return alias; + } + + private AliasList buildAreaAliasList(Collection areaAliases){ + AliasList aliasList = new AliasList(); + if (areaAliases != null) { + for (Object o : areaAliases) { + Alias alias = new Alias(); + alias.setContent(Objects.toString(o, null)); + aliasList.getAlias().add(alias); + } + } + return aliasList; + } + + private DefAreaElementInner buildArea(SolrDocument doc) { + DefAreaElementInner Area = new DefAreaElementInner(); + Area.setName(Objects.toString(doc.get("area_name"), null)); + Area.setId(Objects.toString(doc.get("area_gid"), null)); + Area.setTypeId(Objects.toString(doc.get("area_type_gid"), null)); + Area.setType(Objects.toString(doc.get("area_type_name"), null)); + Area.setLifeSpan(buildLifeSpan(doc.get("area_begindate"), doc.get("area_enddate"), doc.get("area_ended"))); + Area.setAliasList(buildAreaAliasList(doc.getFieldValues("area_aliases_name"))); + + return Area; + } + + private DefAreaElementInner buildEndArea(SolrDocument doc) { + DefAreaElementInner endArea = new DefAreaElementInner(); + endArea.setName(Objects.toString(doc.get("endarea_name"), null)); + endArea.setId(Objects.toString(doc.get("endarea_gid"), null)); + endArea.setTypeId(Objects.toString(doc.get("endarea_type_gid"), null)); + endArea.setType(Objects.toString(doc.get("endarea_type_name"), null)); + endArea.setLifeSpan(buildLifeSpan(doc.get("endarea_begindate"), doc.get("endarea_enddate"), doc.get("endarea_ended"))); + endArea.setAliasList(buildAreaAliasList(doc.getFieldValues("endarea_aliases_name"))); + + return endArea; + } + + private DefAreaElementInner buildBeginArea(SolrDocument doc) { + DefAreaElementInner beginArea = new DefAreaElementInner(); + beginArea.setName(Objects.toString(doc.get("beginarea_name"), null)); + beginArea.setId(Objects.toString(doc.get("beginarea_gid"), null)); + beginArea.setTypeId(Objects.toString(doc.get("beginarea_type_gid"), null)); + beginArea.setType(Objects.toString(doc.get("beginarea_type_name"), null)); + beginArea.setLifeSpan(buildLifeSpan(doc.get("beginarea_begindate"), doc.get("beginarea_enddate"), doc.get("beginarea_ended"))); + beginArea.setAliasList(buildAreaAliasList(doc.getFieldValues("beginarea_aliases_name"))); + + return beginArea; + } + + private TagList buildTagList(SolrDocument doc) { + TagList tagList = new TagList(); + List tagNames = toStringList(doc.getFieldValues("tag_name")); + List tagCounts = toStringList(doc.getFieldValues("tag_count")); + for (int i = 0; i < tagNames.size(); i++){ + Tag tag = new Tag(); + tag.setName(tagNames.get(i)); + tag.setCount(BigInteger.valueOf(Long.parseLong(tagCounts.get(i)))); + tagList.getTag().add(tag); + } + return tagList; + + } + + private IpiList buildIpiList(SolrDocument doc) { + IpiList ipiList = new IpiList(); + Collection raw_ipi_list = doc.getFieldValues("ipi"); + if (raw_ipi_list != null) { + for (Object o : raw_ipi_list) { + ipiList.getIpi().add(Objects.toString(o, null)); + } + } + return ipiList; + } + + private IsniList buildIsniList(SolrDocument doc) { + IsniList isniList = new IsniList(); + Collection raw_isni_list = doc.getFieldValues("isni"); + if (raw_isni_list != null) { + for (Object o : raw_isni_list) { + isniList.getIsni().add(Objects.toString(o, null)); + } + } + return isniList; + } + private List toStringList(Collection coll) { + List list = new ArrayList<>(); + if (coll != null) { + for (Object obj : coll) { + list.add(Objects.toString(obj, null)); + } + } + return list; + } +} \ No newline at end of file diff --git a/mbsssss b/mbsssss index a8f2a72..8319a05 160000 --- a/mbsssss +++ b/mbsssss @@ -1 +1 @@ -Subproject commit a8f2a72d734432a781a5bc9d3c0df99eff6210a1 +Subproject commit 8319a050a78640d8ce7874e1fe9bbab941abb7f0