package com.homme.demo.service; import java.io.ByteArrayInputStream; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ExecutionException; import org.apache.poi.xwpf.usermodel.XWPFDocument; import org.apache.poi.xwpf.usermodel.XWPFParagraph; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.text.PDFTextStripper; import org.apache.poi.hwpf.HWPFDocument; import org.apache.poi.hwpf.extractor.WordExtractor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.ByteArrayResource; import org.springframework.http.MediaType; import org.springframework.http.ResponseCookie; import org.springframework.http.ResponseEntity; import org.springframework.http.client.MultipartBodyBuilder; import org.springframework.stereotype.Service; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.reactive.function.BodyInserters; import org.springframework.web.reactive.function.client.ClientResponse; import org.springframework.web.reactive.function.client.ExchangeStrategies; import org.springframework.web.reactive.function.client.WebClient; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.homme.demo.dto.ApiResponse; import com.homme.demo.dto.FileProcessedDto; import com.homme.demo.dto.UploadWordFileResponseDto; import reactor.core.publisher.Mono; @Service public class KnowledgeIngestService { @Value("${humbee.login.email}") private String email; @Value("${humbee.login.password}") private String password; @Value("${openai.enabled}") private boolean openAiEnabled; private final WebClient webClient; private final MultiValueMap sessionCookies = new LinkedMultiValueMap<>(); @Autowired private DocumentStorageService documentStorageService; @Autowired private PromptService promptService; public KnowledgeIngestService () { ExchangeStrategies strategies = ExchangeStrategies.builder() .codecs(configurer -> configurer.defaultCodecs().maxInMemorySize(10 * 1024 * 1024)) .build(); this.webClient = WebClient.builder() .baseUrl("https://cloud.humbee.de") .exchangeStrategies(strategies) .build(); } public List> buildKnowledgeBase() throws Exception{ login(); String json = getDiroctoryJson(); List links = extractWordLinks(json); ExecutorService pool = Executors.newFixedThreadPool(6); try { List>> futures = new ArrayList<>(); for(String link :links) { futures.add(pool.submit(() -> processOneFile(link))); } List> result = new ArrayList<>(); for(Future> f: futures) { result.add(f.get()); } return result; } catch (InterruptedException | ExecutionException e) { Thread.currentThread().interrupt(); throw new RuntimeException("Fehler bei paralleler Dateiverarbeitung: " + e.getMessage()); } finally { pool.shutdown(); } } public ApiResponse processOneFile(String link){ try { String rawText = readWordFile(link); String cleanText = openAiEnabled ? promptService.removeSpeakerLabelsAndTimestamps(rawText) : rawText; ResponseEntity saveResponse = documentStorageService.saveDocuemntWithChuncks(link, rawText, cleanText); String statusResponse = saveResponse.getStatusCode().toString(); FileProcessedDto fileProcessedDto = FileProcessedDto.builder() .link(link) .status(statusResponse) .build(); return ApiResponse.ok(fileProcessedDto); }catch (Exception e ) { return ApiResponse.error("Fehler bei "+ link + ": "+ e.getMessage()); } } public void login() { String loginPath = "/account/login"; sessionCookies.clear(); Map requestBody = new HashMap<>(); requestBody.put("email", email); requestBody.put("password", password); ClientResponse response = webClient.post() .uri(loginPath) .contentType(MediaType.APPLICATION_JSON) .accept(MediaType.ALL) .bodyValue(requestBody) .exchangeToMono(res -> Mono.just(res)) .block(); if(response == null) { throw new RuntimeException("Anmeldung fehlgeschlagen: Keine Antwort vom Server."); } if(!(response.statusCode().is2xxSuccessful() || response.statusCode().is3xxRedirection())) { throw new RuntimeException("Anmeldung fehlgeschlagen. Status: " + response.statusCode().value()); } Map> cookiesMap = response.cookies(); for(String name : cookiesMap.keySet()) { List values = cookiesMap.get(name); for(ResponseCookie cookie : values) { sessionCookies.add(name, cookie.getValue()); } } if(sessionCookies.isEmpty()) { throw new RuntimeException("Anmeldung fehlgeschlagen: Keine Cookies vom Server erhalten."); } } public String getDiroctoryJson() { ensureLoggedIn(); String responseBody = webClient.get() .uri("/directory/a18eb2ff-d045-4926-8707-322be2588aab/content") .accept(MediaType.APPLICATION_JSON) .cookies(cookies -> cookies.addAll(sessionCookies)) .retrieve() .bodyToMono(String.class) .block(); if(responseBody == null || responseBody.isEmpty()) { throw new RuntimeException("Das Verzeichnis konnte nicht geladen werden oder ist leer."); } return responseBody; } public List extractWordLinks(String json) throws Exception{ List links = new ArrayList<>(); ObjectMapper objectMapper = new ObjectMapper(); JsonNode root = objectMapper.readTree(json); JsonNode items = root.get("items"); if(items == null || !items.isArray()) { return links; } for(JsonNode item:items) { JsonNode quickActions = item.path("_actions").path("quickActions"); for(JsonNode action: quickActions) { String id = action.path("id").asText(); String href = action.path("href").asText(); if("download".equals(id) && (href.contains(".docx?") || href.contains(".doc?") || href.contains(".txt"))) { links.add("https://cloud.humbee.de"+href); } } } return links; } public String readWordFile(String downloadHref) throws Exception{ ensureLoggedIn(); if(downloadHref == null || downloadHref.isBlank()) { throw new RuntimeException("Der Download-Link ist leer oder ungültig."); } byte[] fileBytes = webClient.get() .uri(downloadHref) .cookies(cookies -> cookies.addAll(sessionCookies)) .retrieve() .bodyToMono(byte[].class) .block(); return extractText(fileBytes, downloadHref); } public String extractText(byte[] fileBytes, String fileName) throws Exception{ String name = (fileName == null) ? "" : fileName.toLowerCase(); if(name.contains(".docx")) { try(XWPFDocument document = new XWPFDocument(new ByteArrayInputStream(fileBytes))) { StringBuilder text = new StringBuilder(); for(XWPFParagraph paragraph : document.getParagraphs()) { text.append(paragraph.getText()).append("\n"); } return text.toString(); }catch(Exception e) { throw new RuntimeException("Die DDC-Datei konnte nicht gelesen werden."); } }else if(name.contains(".doc")) { try(HWPFDocument document = new HWPFDocument(new ByteArrayInputStream(fileBytes)); WordExtractor extractor = new WordExtractor(document)){ return extractor.getText(); } } else if(name.endsWith(".pdf")) { try(PDDocument doc = PDDocument.load(fileBytes)){ String rawText= new PDFTextStripper().getText(doc); return rawText; } } else if(name.contains(".txt")){ return decodeBytes(fileBytes); } throw new RuntimeException("Nicht unterstützer Dateityp"); } private void ensureLoggedIn() { if(sessionCookies.isEmpty()) { throw new RuntimeException("Nicht angeledet. Bitte zuerst einloggen."); } } public String decodeBytes(byte[] bytes) { if (bytes == null || bytes.length == 0) return ""; if (bytes.length >= 3 && (bytes[0]&0xFF)==0xEF && (bytes[1]&0xFF)==0xBB && (bytes[2]&0xFF)==0xBF) return new String(bytes, 3, bytes.length-3, java.nio.charset.StandardCharsets.UTF_8); if (bytes.length >= 2 && (bytes[0]&0xFF)==0xFF && (bytes[1]&0xFF)==0xFE) return new String(bytes, 2, bytes.length-2, java.nio.charset.StandardCharsets.UTF_16LE); if (bytes.length >= 2 && (bytes[0]&0xFF)==0xFE && (bytes[1]&0xFF)==0xFF) return new String(bytes, 2, bytes.length-2, java.nio.charset.StandardCharsets.UTF_16BE); int nulls = 0; for (int i = 1; i < Math.min(bytes.length, 400); i += 2) if (bytes[i] == 0) nulls++; if (nulls > 50) return new String(bytes, java.nio.charset.StandardCharsets.UTF_16LE); return new String(bytes, java.nio.charset.StandardCharsets.UTF_8); } public UploadWordFileResponseDto uploadWordFile(String fileName, byte[] docxBytes) throws Exception{ login(); MultipartBodyBuilder builder = new MultipartBodyBuilder(); builder.part("file", new ByteArrayResource(docxBytes)) .filename(fileName) .contentType(MediaType.parseMediaType( "application/vnd.openxmlformats-officedocument.wordprocessingml.document")); String response = webClient.post() .uri("https://cloud.humbee.de/directory/a18eb2ff-d045-4926-8707-322be2588aab/content") .cookies(c -> c.addAll(sessionCookies)) .contentType(MediaType.MULTIPART_FORM_DATA) .body(BodyInserters.fromMultipartData(builder.build())) .retrieve() .bodyToMono(String.class) .block(); System.out.println(" Humbee Upload Antwort: "+ response); String json = getDiroctoryJson(); List links = extractWordLinks(json); for(String link: links) { if(link.contains(fileName)) { return UploadWordFileResponseDto.builder() .success(true) .fileName(fileName) .link(link) .message("Word-Datei erfolgreichhochgeladen") .build(); } } return UploadWordFileResponseDto.builder() .success(false) .fileName(fileName) .link(null) .message("Hochgeladen, aber im Verzeichnis nicht gefunden") .build(); } }