Skip to content

Commit a840544

Browse files
alex1702alex1702
committed
ard - add compilation subpage handling
zdf: fix sha topic page extension
2 parents 9ca4c62 + 300a17d commit a840544

8 files changed

Lines changed: 147 additions & 31 deletions

File tree

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ group = 'de.mediathekview'
2626
base {
2727
archivesName = "MServer"
2828
}
29-
version = '3.1.286'
29+
version = '3.1.287'
3030

3131
def jarName = 'MServer.jar'
3232
def mainClass = 'mServer.Main'

‎src/main/java/mServer/crawler/sender/ard/ArdConstants.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,13 @@ public class ArdConstants {
1111

1212
public static final String TOPICS_URL = API_URL + "/page-gateway/pages/%s/editorial/experiment-a-z?embedded=false";
1313
public static final String TOPICS_COMPILATION_URL = API_URL + "/page-gateway/widgets/%s/editorials/%s?pageNumber=0&pageSize=%s";
14+
public static final String TOPIC_COMPILATION_PAGE_URL = API_URL + "/page-gateway/widgets/%s/compilation/%s?pageNumber=%d&pageSize=%d";
1415
public static final String TOPIC_URL = API_URL + "/page-gateway/widgets/ard/asset/%s?pageSize=%d";
1516
public static final String DAY_PAGE_URL = "https://programm-api.ard.de/program/api/program?day=%s&channelIds=%s&mode=channel";
1617

1718
public static final int TOPICS_COMPILATION_PAGE_SIZE = 100;
1819
public static final int TOPIC_PAGE_SIZE = 50;
20+
public static final int MAX_COMPILATION_SUB_PAGES = 4;
1921

2022
public static final String DEFAULT_CLIENT = "ard";
2123

‎src/main/java/mServer/crawler/sender/ard/ArdTopicInfoDto.java‎

Lines changed: 48 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -6,53 +6,77 @@
66
public class ArdTopicInfoDto {
77
private final Set<ArdFilmInfoDto> filmInfos;
88
private String id;
9-
private int subPageNumber;
9+
private int pageNumber;
10+
private int pageSize;
11+
private int totalElements;
1012
private int maxSubPageNumber;
1113

14+
1215
public ArdTopicInfoDto(final Set<ArdFilmInfoDto> filmInfos) {
1316
this.filmInfos = filmInfos;
14-
subPageNumber = 0;
15-
maxSubPageNumber = 0;
17+
setPageNumber(0);
18+
setPageSize(0);
19+
setTotalElements(0);
20+
setMaxSubPageNumber(0);
1621
}
1722

1823
public Set<ArdFilmInfoDto> getFilmInfos() {
1924
return filmInfos;
2025
}
2126

22-
public String getId() {
23-
return id;
27+
@Override
28+
public boolean equals(final Object o) {
29+
if (this == o) {
30+
return true;
31+
}
32+
if (!(o instanceof final ArdTopicInfoDto that)) {
33+
return false;
34+
}
35+
return getId() == that.getId()
36+
&& getPageNumber() == that.getPageNumber()
37+
&& getPageSize() == that.getPageSize()
38+
&& getTotalElements() == that.getTotalElements()
39+
&& Objects.equals(filmInfos, that.filmInfos);
2440
}
2541

26-
public void setId(String id) {
27-
this.id = id;
42+
@Override
43+
public int hashCode() {
44+
return Objects.hash(filmInfos, getId(), getPageNumber(), getPageSize(), getTotalElements());
2845
}
2946

30-
public int getSubPageNumber() {
31-
return subPageNumber;
47+
public String getId() { return id; }
48+
49+
public int getPageNumber() {
50+
return pageNumber;
3251
}
3352

34-
public void setSubPageNumber(final int subPageNumber) {
35-
this.subPageNumber = subPageNumber;
53+
public void setPageNumber(int pageNumber) {
54+
this.pageNumber = pageNumber;
3655
}
3756

38-
public int getMaxSubPageNumber() {
39-
return maxSubPageNumber;
57+
public int getPageSize() {
58+
return pageSize;
4059
}
4160

42-
public void setMaxSubPageNumber(final int maxSubPageNumber) {
43-
this.maxSubPageNumber = maxSubPageNumber;
61+
public void setPageSize(int pageSize) {
62+
this.pageSize = pageSize;
4463
}
4564

46-
@Override
47-
public boolean equals(Object o) {
48-
if (this == o) return true;
49-
if (o == null || getClass() != o.getClass()) return false;
50-
ArdTopicInfoDto that = (ArdTopicInfoDto) o;
51-
return Objects.equals(id, that.id) && subPageNumber == that.subPageNumber && maxSubPageNumber == that.maxSubPageNumber && Objects.equals(filmInfos, that.filmInfos);
65+
public int getTotalElements() {
66+
return totalElements;
5267
}
5368

54-
@Override
55-
public int hashCode() {
56-
return Objects.hash(id, filmInfos, subPageNumber, maxSubPageNumber);
69+
public void setTotalElements(int totalElements) {
70+
this.totalElements = totalElements;
71+
}
72+
73+
public void setId(String id) { this.id = id; }
74+
75+
public int getMaxSubPageNumber() {
76+
return maxSubPageNumber;
77+
}
78+
79+
public void setMaxSubPageNumber(final int maxSubPageNumber) {
80+
this.maxSubPageNumber = maxSubPageNumber;
5781
}
5882
}

‎src/main/java/mServer/crawler/sender/ard/json/ArdTopicCompilationDeserializer.java‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,19 +33,39 @@ public ArdTopicInfoDto deserialize(
3333
results.addAll(parseTeasers(teasers));
3434
}
3535
});
36+
} else if (showPageObject.getAsJsonObject().has(ELEMENT_TEASERS)) {
37+
final JsonArray teasers = showPageObject.getAsJsonObject().get(ELEMENT_TEASERS).getAsJsonArray();
38+
results.addAll(parseTeasers(teasers));
3639
}
3740

38-
final JsonElement paginationElement = showPageObject.get(ELEMENT_PAGINATION);
41+
final JsonElement paginationElement = findPaginationElement(showPageObject);
3942
final int pageNumber = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_NUMBER);
4043
final int totalElements = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_TOTAL_ELEMENTS);
44+
ardTopicInfoDto.setTotalElements(totalElements);
4145
final int pageSize = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_SIZE);
46+
ardTopicInfoDto.setPageSize(pageSize);
4247
ardTopicInfoDto.setMaxSubPageNumber(pageSize == 0 ? 0 :
4348
(totalElements + pageSize - 1) / pageSize);
44-
ardTopicInfoDto.setSubPageNumber(pageNumber);
49+
ardTopicInfoDto.setPageNumber(pageNumber);
4550

4651
return ardTopicInfoDto;
4752
}
4853

54+
private JsonElement findPaginationElement(JsonObject showPageObject) {
55+
if (showPageObject.has(ELEMENT_WIDGETS)) {
56+
final JsonElement widgetElement = showPageObject.get(ELEMENT_WIDGETS);
57+
if (widgetElement.isJsonArray()) {
58+
final JsonArray widgetArray = widgetElement.getAsJsonArray();
59+
if (!widgetArray.isEmpty()) {
60+
return widgetArray.get(0).getAsJsonObject().get(ELEMENT_PAGINATION);
61+
}
62+
}
63+
} else if (showPageObject.has(ELEMENT_PAGINATION)) {
64+
return showPageObject.get(ELEMENT_PAGINATION);
65+
}
66+
return null;
67+
}
68+
4969
private int getChildElementAsIntOrNullIfNotExist(
5070
final JsonElement parentElement, final String childElementName) {
5171
if (parentElement == null || parentElement.isJsonNull()) {

‎src/main/java/mServer/crawler/sender/ard/json/ArdTopicPageDeserializer.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,12 @@ public ArdTopicInfoDto deserialize(
3535
id.ifPresent(ardTopicInfoDto::setId);
3636

3737
final JsonElement paginationElement = showPageObject.get(ELEMENT_PAGINATION);
38-
ardTopicInfoDto.setSubPageNumber(
38+
ardTopicInfoDto.setPageNumber(
3939
getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_NUMBER));
4040
final int totalElements = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_TOTAL_ELEMENTS);
41+
ardTopicInfoDto.setTotalElements(totalElements);
4142
final int pageSize = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_SIZE);
43+
ardTopicInfoDto.setPageSize(pageSize);
4244
ardTopicInfoDto.setMaxSubPageNumber(pageSize == 0 ? 0 :
4345
(totalElements + pageSize - 1) / pageSize);
4446

‎src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicCompilationTask.java‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,28 @@
33
import com.google.gson.reflect.TypeToken;
44
import jakarta.ws.rs.client.WebTarget;
55
import mServer.crawler.sender.MediathekReader;
6+
import mServer.crawler.sender.ard.ArdConstants;
67
import mServer.crawler.sender.ard.ArdFilmInfoDto;
78
import mServer.crawler.sender.ard.ArdTopicInfoDto;
89
import mServer.crawler.sender.ard.json.ArdTopicCompilationDeserializer;
910
import mServer.crawler.sender.base.AbstractRecursivConverterTask;
1011
import mServer.crawler.sender.base.CrawlerUrlDTO;
12+
import org.apache.logging.log4j.LogManager;
13+
import org.apache.logging.log4j.Logger;
1114

1215
import java.lang.reflect.Type;
1316
import java.util.concurrent.ConcurrentLinkedQueue;
17+
import java.util.regex.Matcher;
18+
import java.util.regex.Pattern;
1419

1520
public class ArdTopicCompilationTask extends ArdTaskBase<ArdFilmInfoDto, CrawlerUrlDTO> {
21+
private static final Logger LOG = LogManager.getLogger(ArdTopicCompilationTask.class);
22+
private static final Pattern UUID_PATTERN = Pattern.compile("/compilation/([^/?]+)([/?])");
23+
private static final Pattern WIDGETS_SENDER_PATTERN = Pattern.compile("/pages/([^/]+)/"); // captures the segment after /widgets/ (e.g. "ard")
1624

1725
private static final Type ARDTOPICINFODTO_TYPE_TOKEN =
18-
new TypeToken<ArdTopicInfoDto>() {}.getType();
26+
new TypeToken<ArdTopicInfoDto>() {
27+
}.getType();
1928

2029
public ArdTopicCompilationTask(
2130
final MediathekReader aCrawler, final ConcurrentLinkedQueue<CrawlerUrlDTO> aUrlToCrawlDtos) {
@@ -31,7 +40,52 @@ protected void processRestTarget(final CrawlerUrlDTO aDTO, final WebTarget aTarg
3140
&& topicInfo.getFilmInfos() != null
3241
&& !topicInfo.getFilmInfos().isEmpty()) {
3342
taskResults.addAll(topicInfo.getFilmInfos());
43+
44+
final ConcurrentLinkedQueue<CrawlerUrlDTO> subpages = createSubPageUrls(aDTO, topicInfo);
45+
if (!subpages.isEmpty()) {
46+
taskResults.addAll(createNewOwnInstance(subpages).fork().join());
47+
}
48+
}
49+
}
50+
51+
private ConcurrentLinkedQueue<CrawlerUrlDTO> createSubPageUrls(CrawlerUrlDTO urlDto, ArdTopicInfoDto topicInfo) {
52+
final ConcurrentLinkedQueue<CrawlerUrlDTO> subpages = new ConcurrentLinkedQueue<>();
53+
if (topicInfo.getTotalElements() < topicInfo.getPageSize() || topicInfo.getPageNumber() > 0) {
54+
return subpages;
55+
}
56+
//
57+
final String sender = extractSender(urlDto.getUrl());
58+
final String id = extractCompilation(urlDto.getUrl());
59+
60+
int index = 0;
61+
while ((topicInfo.getPageSize() + (index * topicInfo.getPageSize())) < topicInfo.getTotalElements()) {
62+
subpages.add(new CrawlerUrlDTO(String.format(ArdConstants.TOPIC_COMPILATION_PAGE_URL, sender, id, index + 1, topicInfo.getPageSize())));
63+
index++;
64+
if (index >= ArdConstants.MAX_COMPILATION_SUB_PAGES) {
65+
LOG.debug("ignore more subpage due to limit of {} pages but found {}", ArdConstants.MAX_COMPILATION_SUB_PAGES, Integer.valueOf(topicInfo.getTotalElements() / topicInfo.getPageSize()));
66+
break;
67+
}
68+
}
69+
return subpages;
70+
}
71+
72+
private String extractCompilation(String url) {
73+
final Matcher m = UUID_PATTERN.matcher(url);
74+
if (m.find()) {
75+
return m.group(1);
76+
}
77+
78+
LOG.warn("no compilation pattern found in url {}", url);
79+
return "";
80+
}
81+
82+
private String extractSender(String url) {
83+
final Matcher m = WIDGETS_SENDER_PATTERN.matcher(url);
84+
if (m.find()) {
85+
return m.group(1);
3486
}
87+
LOG.warn("no sender pattern found in url {}", url);
88+
return "";
3589
}
3690

3791
@Override

‎src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,20 @@ public class ArdTopicPageTask extends ArdTaskBase<ArdFilmInfoDto, CrawlerUrlDTO>
3434
private static final Set<String> TOPICS_LOAD_ALL_PAGES = new HashSet<>();
3535

3636
static {
37+
// Retro
38+
TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3JhZGlvYnJlbWVuLmRlL2IwYTJlZWFlLWI2NjAtNDI5Yi05ZTE3LTM5YzlkZDhmNTc4Ng");
39+
TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2JyLmRlL2Jyb2FkY2FzdFNlcmllcy8yOGEwMzU4Yi00N2ViLTQ0MDktOGFmZi02ZjVkMDE5NDA2NDc");
40+
TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3dkci5kZS93ZHJyZXRybw");
41+
TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL25kci5kZS80NTg4");
42+
TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3JiYi1vbmxpbmUuZGUvYmVybGluZXItYWJlbmRzY2hhdQ");
43+
TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2hyLW9ubGluZS8zODIyMDExOA");
44+
TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3N3ci5kZS8yNDEwMzE1Ng");
45+
TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2hyLW9ubGluZS8zODIyMDA5Ng");
46+
TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3NyLW9ubGluZS5kZS9SRVRSTy1IRA");
47+
// NDR Nordstory
48+
TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL25kci5kZS8xNDk1");
49+
// SWR Eisenbahn-Romantik
50+
TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3N3ci5kZS8yMjUwOTg2");
3751
// example: TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2Rhc2Vyc3RlLmRlL3Nwb3J0c2NoYXU");
3852
}
3953

@@ -68,7 +82,7 @@ private ConcurrentLinkedQueue<CrawlerUrlDTO> createSubPageUrls(
6882
final WebTarget aTarget, final ArdTopicInfoDto topicInfo) {
6983
final ConcurrentLinkedQueue<CrawlerUrlDTO> subpages = new ConcurrentLinkedQueue<>();
7084

71-
final int actualSubPageNumber = topicInfo.getSubPageNumber();
85+
final int actualSubPageNumber = topicInfo.getPageNumber();
7286
final int maximumAllowedSubpages = getMaximumSubpages(topicInfo.getId());
7387
if (actualSubPageNumber != 0) {
7488
LOG.debug("Sub page {} is already the maximum allowed sub page.", actualSubPageNumber);

‎src/main/java/mServer/crawler/sender/zdf/ZdfConstants.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ public final class ZdfConstants {
3535
public static final String URL_TOPIC_PAGE_VARIABLES = "{\"seasonIndex\":%d,\"episodesPageSize\":%d,\"canonical\":\"%s\",\"sortBy\":[{\"field\":\"EDITORIAL_DATE\",\"direction\":\"DESC\"}]}";
3636
public static final String URL_TOPIC_PAGE_VARIABLES_WITH_CURSOR = "{\"seasonIndex\":%d,\"episodesPageSize\":%d,\"canonical\":\"%s\",\"sortBy\":[{\"field\":\"EDITORIAL_DATE\",\"direction\":\"DESC\"}],\"episodesAfter\":\"%s\"}";
3737
public static final String URL_TOPIC_PAGE_EXTENSIONS =
38-
"{\"persistedQuery\":{\"version\":1,\"sha256Hash\":\"9412a0f4ac55dc37d46975d461ec64bfd14380d815df843a1492348f77b5c99a\"}}";
38+
"{\"persistedQuery\":{\"version\":1,\"sha256Hash\":\"81237cafa2f0176d351b21bff20cdcb5e5755092a0bd42d7271018c5440a4493\"}}";
3939

4040
public static final String URL_TOPIC_PAGE_NO_SEASON = URL_API_BASE + "/graphql?operationName=getMetaCollectionContent&" +
4141
"variables=%s&" +

0 commit comments

Comments
 (0)