Mohammad Zwaib
376 lines
10 KiB
Java
376 lines
10 KiB
Java
package com.homme.demo.service;
|
|
|
|
|
|
import java.io.ByteArrayInputStream;
|
|
import java.util.ArrayList;
|
|
import java.util.HashMap;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.concurrent.ExecutorService;
|
|
import java.util.concurrent.Executors;
|
|
import java.util.concurrent.Future;
|
|
import java.util.concurrent.ExecutionException;
|
|
|
|
import org.apache.poi.xwpf.usermodel.XWPFDocument;
|
|
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
|
|
import org.apache.pdfbox.pdmodel.PDDocument;
|
|
import org.apache.pdfbox.text.PDFTextStripper;
|
|
import org.apache.poi.hwpf.HWPFDocument;
|
|
import org.apache.poi.hwpf.extractor.WordExtractor;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.core.io.ByteArrayResource;
|
|
import org.springframework.http.MediaType;
|
|
import org.springframework.http.ResponseCookie;
|
|
import org.springframework.http.ResponseEntity;
|
|
import org.springframework.http.client.MultipartBodyBuilder;
|
|
import org.springframework.stereotype.Service;
|
|
import org.springframework.util.LinkedMultiValueMap;
|
|
import org.springframework.util.MultiValueMap;
|
|
import org.springframework.web.reactive.function.BodyInserters;
|
|
import org.springframework.web.reactive.function.client.ClientResponse;
|
|
import org.springframework.web.reactive.function.client.ExchangeStrategies;
|
|
import org.springframework.web.reactive.function.client.WebClient;
|
|
|
|
|
|
import com.fasterxml.jackson.databind.JsonNode;
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
import com.homme.demo.dto.ApiResponse;
|
|
import com.homme.demo.dto.FileProcessedDto;
|
|
import com.homme.demo.dto.UploadWordFileResponseDto;
|
|
import reactor.core.publisher.Mono;
|
|
|
|
|
|
|
|
|
|
@Service
|
|
public class KnowledgeIngestService {
|
|
|
|
@Value("${humbee.login.email}")
|
|
private String email;
|
|
|
|
@Value("${humbee.login.password}")
|
|
private String password;
|
|
|
|
@Value("${openai.enabled}")
|
|
private boolean openAiEnabled;
|
|
|
|
private final WebClient webClient;
|
|
|
|
private final MultiValueMap<String, String> sessionCookies = new LinkedMultiValueMap<>();
|
|
|
|
@Autowired
|
|
private DocumentStorageService documentStorageService;
|
|
|
|
@Autowired
|
|
private PromptService promptService;
|
|
|
|
public KnowledgeIngestService () {
|
|
|
|
|
|
ExchangeStrategies strategies = ExchangeStrategies.builder()
|
|
.codecs(configurer -> configurer.defaultCodecs().maxInMemorySize(10 * 1024 * 1024))
|
|
.build();
|
|
|
|
this.webClient = WebClient.builder()
|
|
.baseUrl("https://cloud.humbee.de")
|
|
.exchangeStrategies(strategies)
|
|
.build();
|
|
|
|
}
|
|
public List<ApiResponse<FileProcessedDto>> buildKnowledgeBase() throws Exception{
|
|
|
|
login();
|
|
String json = getDiroctoryJson();
|
|
List<String> links = extractWordLinks(json);
|
|
|
|
ExecutorService pool = Executors.newFixedThreadPool(6);
|
|
try {
|
|
|
|
List<Future<ApiResponse<FileProcessedDto>>> futures = new ArrayList<>();
|
|
|
|
for(String link :links) {
|
|
|
|
futures.add(pool.submit(() -> processOneFile(link)));
|
|
|
|
}
|
|
List<ApiResponse<FileProcessedDto>> result = new ArrayList<>();
|
|
|
|
for(Future<ApiResponse<FileProcessedDto>> f: futures) {
|
|
|
|
result.add(f.get());
|
|
}
|
|
return result;
|
|
|
|
|
|
} catch (InterruptedException | ExecutionException e) {
|
|
Thread.currentThread().interrupt();
|
|
throw new RuntimeException("Fehler bei paralleler Dateiverarbeitung: " + e.getMessage());
|
|
} finally {
|
|
|
|
pool.shutdown();
|
|
}
|
|
|
|
|
|
}
|
|
public ApiResponse<FileProcessedDto> processOneFile(String link){
|
|
|
|
|
|
try {
|
|
String rawText = readWordFile(link);
|
|
|
|
|
|
String cleanText = openAiEnabled
|
|
? promptService.removeSpeakerLabelsAndTimestamps(rawText)
|
|
: rawText;
|
|
|
|
|
|
ResponseEntity<String> saveResponse = documentStorageService.saveDocuemntWithChuncks(link, rawText, cleanText);
|
|
|
|
|
|
String statusResponse = saveResponse.getStatusCode().toString();
|
|
FileProcessedDto fileProcessedDto = FileProcessedDto.builder()
|
|
.link(link)
|
|
.status(statusResponse)
|
|
.build();
|
|
|
|
return ApiResponse.ok(fileProcessedDto);
|
|
|
|
|
|
}catch (Exception e ) {
|
|
|
|
return ApiResponse.error("Fehler bei "+ link + ": "+ e.getMessage());
|
|
}
|
|
|
|
}
|
|
|
|
public void login() {
|
|
|
|
String loginPath = "/account/login";
|
|
sessionCookies.clear();
|
|
Map<String, String> requestBody = new HashMap<>();
|
|
requestBody.put("email", email);
|
|
requestBody.put("password", password);
|
|
|
|
ClientResponse response = webClient.post()
|
|
.uri(loginPath)
|
|
.contentType(MediaType.APPLICATION_JSON)
|
|
.accept(MediaType.ALL)
|
|
.bodyValue(requestBody)
|
|
.exchangeToMono(res -> Mono.just(res))
|
|
.block();
|
|
|
|
if(response == null) {
|
|
throw new RuntimeException("Anmeldung fehlgeschlagen: Keine Antwort vom Server.");
|
|
}
|
|
|
|
if(!(response.statusCode().is2xxSuccessful() || response.statusCode().is3xxRedirection())) {
|
|
throw new RuntimeException("Anmeldung fehlgeschlagen. Status: " + response.statusCode().value());
|
|
}
|
|
|
|
Map<String, List<ResponseCookie>> cookiesMap = response.cookies();
|
|
|
|
for(String name : cookiesMap.keySet()) {
|
|
|
|
List<ResponseCookie> values = cookiesMap.get(name);
|
|
|
|
for(ResponseCookie cookie : values) {
|
|
|
|
sessionCookies.add(name, cookie.getValue());
|
|
}
|
|
}
|
|
if(sessionCookies.isEmpty()) {
|
|
throw new RuntimeException("Anmeldung fehlgeschlagen: Keine Cookies vom Server erhalten.");
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
public String getDiroctoryJson() {
|
|
|
|
ensureLoggedIn();
|
|
String responseBody = webClient.get()
|
|
.uri("/directory/a18eb2ff-d045-4926-8707-322be2588aab/content")
|
|
.accept(MediaType.APPLICATION_JSON)
|
|
.cookies(cookies -> cookies.addAll(sessionCookies))
|
|
.retrieve()
|
|
.bodyToMono(String.class)
|
|
.block();
|
|
|
|
if(responseBody == null || responseBody.isEmpty()) {
|
|
throw new RuntimeException("Das Verzeichnis konnte nicht geladen werden oder ist leer.");
|
|
}
|
|
|
|
|
|
return responseBody;
|
|
|
|
|
|
|
|
}
|
|
public List<String> extractWordLinks(String json) throws Exception{
|
|
|
|
List<String> links = new ArrayList<>();
|
|
ObjectMapper objectMapper = new ObjectMapper();
|
|
JsonNode root = objectMapper.readTree(json);
|
|
JsonNode items = root.get("items");
|
|
if(items == null || !items.isArray()) {
|
|
return links;
|
|
}
|
|
|
|
for(JsonNode item:items) {
|
|
|
|
JsonNode quickActions = item.path("_actions").path("quickActions");
|
|
|
|
for(JsonNode action: quickActions) {
|
|
String id = action.path("id").asText();
|
|
String href = action.path("href").asText();
|
|
|
|
if("download".equals(id) && (href.contains(".docx?") || href.contains(".doc?") || href.contains(".txt"))) {
|
|
links.add("https://cloud.humbee.de"+href);
|
|
|
|
}
|
|
|
|
}
|
|
}
|
|
return links;
|
|
}
|
|
|
|
|
|
public String readWordFile(String downloadHref) throws Exception{
|
|
|
|
ensureLoggedIn();
|
|
if(downloadHref == null || downloadHref.isBlank()) {
|
|
throw new RuntimeException("Der Download-Link ist leer oder ungültig.");
|
|
}
|
|
byte[] fileBytes = webClient.get()
|
|
.uri(downloadHref)
|
|
.cookies(cookies -> cookies.addAll(sessionCookies))
|
|
.retrieve()
|
|
.bodyToMono(byte[].class)
|
|
.block();
|
|
|
|
return extractText(fileBytes, downloadHref);
|
|
}
|
|
|
|
public String extractText(byte[] fileBytes, String fileName) throws Exception{
|
|
|
|
String name = (fileName == null) ? "" : fileName.toLowerCase();
|
|
|
|
if(name.contains(".docx")) {
|
|
try(XWPFDocument document = new XWPFDocument(new ByteArrayInputStream(fileBytes))) {
|
|
|
|
StringBuilder text = new StringBuilder();
|
|
for(XWPFParagraph paragraph : document.getParagraphs()) {
|
|
text.append(paragraph.getText()).append("\n");
|
|
|
|
}
|
|
|
|
return text.toString();
|
|
|
|
}catch(Exception e) {
|
|
throw new RuntimeException("Die DDC-Datei konnte nicht gelesen werden.");
|
|
}
|
|
|
|
}else if(name.contains(".doc")) {
|
|
try(HWPFDocument document = new HWPFDocument(new ByteArrayInputStream(fileBytes));
|
|
WordExtractor extractor = new WordExtractor(document)){
|
|
return extractor.getText();
|
|
|
|
}
|
|
}
|
|
|
|
else if(name.endsWith(".pdf")) {
|
|
|
|
try(PDDocument doc = PDDocument.load(fileBytes)){
|
|
String rawText= new PDFTextStripper().getText(doc);
|
|
return rawText;
|
|
}
|
|
}
|
|
else if(name.contains(".txt")){
|
|
|
|
return decodeBytes(fileBytes);
|
|
|
|
}
|
|
throw new RuntimeException("Nicht unterstützer Dateityp");
|
|
|
|
}
|
|
|
|
private void ensureLoggedIn() {
|
|
if(sessionCookies.isEmpty()) {
|
|
throw new RuntimeException("Nicht angeledet. Bitte zuerst einloggen.");
|
|
}
|
|
}
|
|
|
|
public String decodeBytes(byte[] bytes) {
|
|
if (bytes == null || bytes.length == 0) return "";
|
|
|
|
|
|
if (bytes.length >= 3 && (bytes[0]&0xFF)==0xEF && (bytes[1]&0xFF)==0xBB && (bytes[2]&0xFF)==0xBF)
|
|
return new String(bytes, 3, bytes.length-3, java.nio.charset.StandardCharsets.UTF_8);
|
|
|
|
if (bytes.length >= 2 && (bytes[0]&0xFF)==0xFF && (bytes[1]&0xFF)==0xFE)
|
|
return new String(bytes, 2, bytes.length-2, java.nio.charset.StandardCharsets.UTF_16LE);
|
|
|
|
if (bytes.length >= 2 && (bytes[0]&0xFF)==0xFE && (bytes[1]&0xFF)==0xFF)
|
|
return new String(bytes, 2, bytes.length-2, java.nio.charset.StandardCharsets.UTF_16BE);
|
|
|
|
int nulls = 0;
|
|
for (int i = 1; i < Math.min(bytes.length, 400); i += 2)
|
|
if (bytes[i] == 0) nulls++;
|
|
if (nulls > 50)
|
|
return new String(bytes, java.nio.charset.StandardCharsets.UTF_16LE);
|
|
|
|
return new String(bytes, java.nio.charset.StandardCharsets.UTF_8);
|
|
}
|
|
|
|
public UploadWordFileResponseDto uploadWordFile(String fileName, byte[] docxBytes) throws Exception{
|
|
|
|
login();
|
|
|
|
MultipartBodyBuilder builder = new MultipartBodyBuilder();
|
|
builder.part("file", new ByteArrayResource(docxBytes))
|
|
.filename(fileName)
|
|
.contentType(MediaType.parseMediaType(
|
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.document"));
|
|
|
|
String response = webClient.post()
|
|
.uri("https://cloud.humbee.de/directory/a18eb2ff-d045-4926-8707-322be2588aab/content")
|
|
.cookies(c -> c.addAll(sessionCookies))
|
|
.contentType(MediaType.MULTIPART_FORM_DATA)
|
|
.body(BodyInserters.fromMultipartData(builder.build()))
|
|
.retrieve()
|
|
.bodyToMono(String.class)
|
|
.block();
|
|
|
|
System.out.println(" Humbee Upload Antwort: "+ response);
|
|
|
|
String json = getDiroctoryJson();
|
|
List<String> links = extractWordLinks(json);
|
|
|
|
for(String link: links) {
|
|
|
|
if(link.contains(fileName)) {
|
|
|
|
return UploadWordFileResponseDto.builder()
|
|
.success(true)
|
|
.fileName(fileName)
|
|
.link(link)
|
|
.message("Word-Datei erfolgreichhochgeladen")
|
|
.build();
|
|
}
|
|
|
|
}
|
|
return UploadWordFileResponseDto.builder()
|
|
.success(false)
|
|
.fileName(fileName)
|
|
.link(null)
|
|
.message("Hochgeladen, aber im Verzeichnis nicht gefunden")
|
|
.build();
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|