srf website url

verbessertes Zuschneiden der Beschreibungstexte
ard: search compilations

Author: alex1702

build.gradle

@@ -27,7 +27,7 @@ sourceCompatibility = JavaVersion.VERSION_17
 targetCompatibility = JavaVersion.VERSION_17
 group = 'de.mediathekview'
 archivesBaseName = "MServer"
-version = '3.1.284'
+version = '3.1.285'
 
 def jarName = 'MServer.jar'
 def mainClass = 'mServer.Main'

src/main/java/de/mediathekview/mlib/Const.java

@@ -39,6 +39,7 @@ public class Const {
     public static final int ALTER_FILMLISTE_SEKUNDEN_FUER_AUTOUPDATE = 3 * 60 * 60; // beim Start des Programms wir die Liste geladen wenn sie älter ist als ..
     public static final String TIME_MAX_AGE_FOR_DIFF = "09"; // Uhrzeit ab der die Diffliste alle Änderungen abdeckt, die Filmliste darf also nicht vor xx erstellt worden sein
     public static final int MAX_BESCHREIBUNG = 400; // max länge der Beschreibung in Zeichen -> mehr gibts aber jetzt nicht mehr!
+    public static final int MAX_BESCHREIBUNG_MARGIN_LENGTH = 40;
 
     public static final String DREISAT = "3Sat";
     public static final String ARD = "ARD";

src/main/java/de/mediathekview/mlib/daten/DatenFilm.java

@@ -201,7 +201,7 @@ public static String cleanDescription(String description) {
       description = description.replace("\\\"", "\"");
     }
     if (description.length() > Const.MAX_BESCHREIBUNG) {
-      return description.substring(0, Const.MAX_BESCHREIBUNG) + "\n.....";
+      return SentenceShortener.shorten(description, Const.MAX_BESCHREIBUNG, Const.MAX_BESCHREIBUNG_MARGIN_LENGTH);
     } else {
       return description;
     }

src/main/java/de/mediathekview/mlib/tool/SentenceShortener.java

@@ -0,0 +1,105 @@
+package de.mediathekview.mlib.tool;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+
+public final class SentenceShortener {
+  private static final Pattern WHITESPACE_REGEX = Pattern.compile("\\s+");
+  private static final Pattern SENTENCE_BOUNDARY_REGEX = Pattern.compile("(?<=[.!?])\\s+");
+  private static final String SHORTENING_SUFFIX = "...";
+
+  private SentenceShortener() {
+  }
+
+  /**
+   * Shortens text to a target length while preserving complete sentences.
+   *
+   * <p>Whitespace is normalized before shortening. If the text is already within the target
+   * length, the normalized text is returned unchanged. If adding the next sentence would exceed
+   * {@code targetLength + marginLength}, that sentence is omitted and {@code ...} is appended to
+   * the returned text. If the first sentence alone is already longer than the limit, it is kept
+   * whole.
+   *
+   * @param text the text to shorten; may be {@code null}
+   * @param targetLength the preferred target length; must be positive
+   * @param marginLength the allowed soft overflow beyond the target; must not be negative
+   * @return the shortened text, or {@code null} if the input is {@code null} or blank
+   * @throws IllegalArgumentException if {@code targetLength <= 0} or {@code marginLength < 0}
+   */
+  public static String shorten(String text, int targetLength, int marginLength) {
+    if (targetLength <= 0) {
+      throw new IllegalArgumentException("targetLength must be positive");
+    }
+    if (marginLength < 0) {
+      throw new IllegalArgumentException("marginLength must not be negative");
+    }
+
+    String normalized = normalize(text);
+    if (normalized == null || normalized.length() <= targetLength) {
+      return normalized;
+    }
+
+    int softLimit = targetLength + marginLength;
+    String[] rawSentences = SENTENCE_BOUNDARY_REGEX.split(normalized);
+    List<String> sentences = new ArrayList<>();
+    for (String rawSentence : rawSentences) {
+      String sentence = rawSentence.trim();
+      if (!sentence.isEmpty()) {
+        sentences.add(sentence);
+      }
+    }
+    if (sentences.isEmpty()) {
+      return normalized;
+    }
+
+    List<String> selectedSentences = new ArrayList<>();
+    int currentLength = 0;
+    for (String sentence : sentences) {
+      int candidateLength = selectedSentences.isEmpty()
+              ? sentence.length()
+              : currentLength + 1 + sentence.length();
+      if (candidateLength > softLimit) {
+        break;
+      }
+      selectedSentences.add(sentence);
+      currentLength = candidateLength;
+    }
+
+    if (!selectedSentences.isEmpty()) {
+      String shortened = String.join(" ", selectedSentences);
+      return selectedSentences.size() < sentences.size()
+              ? shortened + SHORTENING_SUFFIX
+              : shortened;
+    }
+    return sentences.size() > 1
+            ? sentences.get(0) + SHORTENING_SUFFIX
+            : sentences.get(0);
+  }
+
+  /**
+   * Shortens text to a target length without any soft overflow margin.
+   *
+   * @param text the text to shorten; may be {@code null}
+   * @param targetLength the preferred target length; must be positive
+   * @return the shortened text, or {@code null} if the input is {@code null} or blank
+   * @throws IllegalArgumentException if {@code targetLength <= 0}
+   */
+  public static String shorten(String text, int targetLength) {
+    return shorten(text, targetLength, 0);
+  }
+
+  /**
+   * Normalizes whitespace and trims a text value.
+   *
+   * @param text the text to normalize; may be {@code null}
+   * @return the normalized text, or {@code null} if the input is {@code null} or blank
+   */
+  public static String normalize(String text) {
+    if (text == null) {
+      return null;
+    }
+    String normalized = WHITESPACE_REGEX.matcher(text).replaceAll(" ").trim();
+    return normalized.isEmpty() ? null : normalized;
+  }
+}

src/main/java/mServer/crawler/sender/ard/ArdCrawler.java

@@ -12,6 +12,7 @@
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ExecutionException;
@@ -27,6 +28,7 @@ public class ArdCrawler extends MediathekCrawler {
           = DateTimeFormatter.ofPattern("yyyy-MM-dd");
 
   public static final String[] MISSING_TOPIC_IDS = new String[]{
+//          "Y3JpZDovL2JyLmRlL2Jyb2FkY2FzdFNlcmllcy9icm9hZGNhc3RTZXJpZXM6L2JyZGUvZmVybnNlaGVuL2JheWVyaXNjaGVzLWZlcm5zZWhlbi9zZW5kdW5nZW4vZGFob2FtLWlzLWRhaG9hbQ"
   };
 
   public ArdCrawler(FilmeSuchen ssearch, int startPrio) {
@@ -131,6 +133,7 @@ private Set<ArdFilmInfoDto> getDaysEntries() throws InterruptedException, Execut
   }
 
   private Set<ArdFilmInfoDto> getTopicsEntries() throws ExecutionException, InterruptedException {
+    final Set<ArdFilmInfoDto> shows = new HashSet<>();
     Set<CrawlerUrlDTO> topics = new HashSet<>();
     topics.addAll(getTopicEntriesBySender(ArdConstants.DEFAULT_CLIENT));
     for (String client : ArdConstants.CLIENTS) {
@@ -149,9 +152,21 @@ private Set<ArdFilmInfoDto> getTopicsEntries() throws ExecutionException, Interr
     ConcurrentLinkedQueue<CrawlerUrlDTO> topicUrls = new ConcurrentLinkedQueue<>(assitUrls);
 
     final ArdTopicPageTask topicTask = new ArdTopicPageTask(this, topicUrls);
-    final Set<ArdFilmInfoDto> filmInfos = forkJoinPool.submit(topicTask).get();
-    Log.sysLog("ard shows by topics: " + filmInfos.size());
-    return filmInfos;
+    final Set<ArdFilmInfoDto> ardFilmInfosWithCompilations = forkJoinPool.submit(topicTask).get();
+
+    // add filmInfos without compilation
+    shows.addAll(ardFilmInfosWithCompilations.stream().filter(filmInfo -> !filmInfo.isCompilation()).toList());
+
+    // search compilations
+    final List<ArdFilmInfoDto> compilations = ardFilmInfosWithCompilations.stream().filter(ArdFilmInfoDto::isCompilation).toList();
+    final ArdTopicCompilationTask compilationTask = new ArdTopicCompilationTask(this, new ConcurrentLinkedQueue<>(compilations));
+    final Set<ArdFilmInfoDto> ardCompilationEntries = forkJoinPool.submit(compilationTask).get();
+
+    final int sizeBefore = shows.size();
+    shows.addAll(ardCompilationEntries.stream().filter(filmInfo -> !filmInfo.isCompilation()).toList());
+    Log.sysLog("ard shows by topics compilation: " + (shows.size() - sizeBefore));
+    Log.sysLog("ard shows by topics: " + shows.size());
+    return shows;
   }
 
   // temporary workaround for missing topics

src/main/java/mServer/crawler/sender/ard/ArdFilmInfoDto.java

@@ -7,18 +7,24 @@ public class ArdFilmInfoDto extends CrawlerUrlDTO {
 
   private final String id;
   private final int numberOfClips;
+  private final boolean isCompilation;
 
-  public ArdFilmInfoDto(String id, String aUrl, int numberOfClips) {
+  public ArdFilmInfoDto(String id, String aUrl, int numberOfClips, boolean isCompilation) {
     super(aUrl);
 
     this.id = id;
     this.numberOfClips = numberOfClips;
+    this.isCompilation = isCompilation;
   }
 
   public String getId() {
     return id;
   }
 
+  public boolean isCompilation() {
+    return isCompilation;
+  }
+
   public int getNumberOfClips() {
     return numberOfClips;
   }
@@ -36,11 +42,12 @@ public boolean equals(Object o) {
     }
     ArdFilmInfoDto that = (ArdFilmInfoDto) o;
     return numberOfClips == that.numberOfClips
-            && Objects.equals(id, that.id);
+            && Objects.equals(id, that.id)
+            && isCompilation == that.isCompilation;
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(super.hashCode(), id, numberOfClips);
+    return Objects.hash(super.hashCode(), id, numberOfClips, isCompilation);
   }
 }

src/main/java/mServer/crawler/sender/ard/WdrM3U8ToMp4Converter.java

@@ -0,0 +1,100 @@
+package mServer.crawler.sender.ard;
+
+import de.mediathekview.mlib.tool.Log;
+import mServer.crawler.sender.base.Qualities;
+
+import java.util.EnumMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * * Converts an m3u8 playlist URL (in the style shown in the example) * into a list of progressive
+ * mp4 URLs. * * Default progressive base host: https://wdr-progressive.ard-mcdn.de * * Example: *
+ * input: *
+ * https://wdrvod-rwrtr.akamaized.net/i/,/media/.../ID_AVC-,270,360,540,720,1080,.mp4.csmil/index-...m3u8
+ * * * produced outputs: * https://wdr-progressive.ard-mcdn.de/media/.../ID_AVC-270.mp4 * ...
+ */
+public class WdrM3U8ToMp4Converter {
+
+  // Looks for /media/...<codec>-<comma-list>.mp4.csmil
+  // group(1) = /media/... (path before codec)
+  // group(2) = _<CODEC>- (codec token + trailing '-')
+  // group(3) = the comma separated tail that contains bitrate numbers (e.g. ",270,360,540,")
+  private static final Pattern MEDIA_PATTERN =
+      Pattern.compile(
+          "(/media/.+?)(_[A-Za-z0-9]+-)([^/]+?)\\.mp4\\.csmil", Pattern.CASE_INSENSITIVE);
+
+  private final String progressiveBase; // no trailing slash
+
+  public WdrM3U8ToMp4Converter() {
+    this("https://wdr-progressive.ard-mcdn.de");
+  }
+
+  public WdrM3U8ToMp4Converter(String progressiveBase) {
+    if (progressiveBase == null)
+      throw new IllegalArgumentException("progressiveBase must not be null");
+    // ensure no trailing slash to make concatenation predictable
+    this.progressiveBase = progressiveBase.replaceAll("/+$", "");
+  }
+
+  /**
+   * Convert an m3u8 url into a list of mp4 URLs.
+   *
+   * @param m3u8Url the source m3u8 url
+   * @return map of mp4 URLs (in the same order as bitrates found)
+   * @throws IllegalArgumentException if the url cannot be parsed or no bitrates found
+   */
+  public Map<Qualities, String> convert(String m3u8Url) {
+    if (m3u8Url == null) throw new IllegalArgumentException("m3u8Url must not be null");
+
+    Map<Qualities, String> result = new EnumMap<>(Qualities.class);
+
+    Matcher m = MEDIA_PATTERN.matcher(m3u8Url);
+    if (!m.find()) {
+      return result;
+    }
+
+    String pathBeforeCodec = m.group(1); // includes leading /media/...
+    String codecWithDash = m.group(2); // e.g. _AVC-
+    String bitrateListPart = m.group(3); // e.g. ",270,360,540,720,1080," or "270,360"
+
+    // build the base path (starts with /media/...)
+    String basePrefix = pathBeforeCodec + codecWithDash; // ends with '-'
+
+    // extract numeric tokens (bitrate values)
+    Pattern digits = Pattern.compile("\\d+");
+    Matcher mDigits = digits.matcher(bitrateListPart);
+
+    while (mDigits.find()) {
+      String bitrate = mDigits.group();
+      // join progressive base + basePrefix + bitrate + .mp4
+      String mp4 = progressiveBase + basePrefix + bitrate + ".mp4";
+      final Qualities resolution = getResolutionFromWidth(bitrate);
+      result.put(resolution, mp4);
+    }
+
+    if (result.isEmpty()) {
+      throw new IllegalArgumentException("No numeric bitrate tokens found in m3u8 URL: " + m3u8Url);
+    }
+
+    return result;
+  }
+
+  private Qualities getResolutionFromWidth(String bitrate) {
+    try {
+      return switch (Integer.parseInt(bitrate)) {
+        case 720 -> Qualities.NORMAL;
+        case 1080 -> Qualities.HD;
+        case 540, 360, 270 -> Qualities.SMALL;
+        default -> {
+          Log.sysLog("Unknown bitrate found in m3u8 URL: " + bitrate + ", defaulting to VERY_SMALL");
+          yield Qualities.SMALL;
+        }
+      };
+    } catch (NumberFormatException e) {
+      Log.errorLog(165346373, e);
+      return Qualities.SMALL;
+    }
+  }
+}

src/main/java/mServer/crawler/sender/ard/json/ArdDayPageDeserializer.java

@@ -50,7 +50,7 @@ private Set<ArdFilmInfoDto> parseChannels(JsonArray channels) {
 
   private ArdFilmInfoDto createFilmInfo(final String id, final int numberOfClips) {
     final String url = String.format(ArdConstants.ITEM_URL, id);
-    return new ArdFilmInfoDto(id, url, numberOfClips);
+    return new ArdFilmInfoDto(id, url, numberOfClips, false);
   }
 
   private Optional<String> toId(final JsonObject teaserObject) {