WordPress
This commit is contained in:
Mohammad Zwaib
@@ -0,0 +1,358 @@
|
||||
package com.homme.demo.service;
|
||||
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
|
||||
import org.apache.poi.xwpf.usermodel.XWPFDocument;
|
||||
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
|
||||
|
||||
import org.apache.poi.hwpf.HWPFDocument;
|
||||
import org.apache.poi.hwpf.extractor.WordExtractor;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseCookie;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.http.client.MultipartBodyBuilder;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.web.reactive.function.BodyInserters;
|
||||
import org.springframework.web.reactive.function.client.ClientResponse;
|
||||
import org.springframework.web.reactive.function.client.ExchangeStrategies;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.homme.demo.dto.ApiResponse;
|
||||
import com.homme.demo.dto.FileProcessedDto;
|
||||
import com.homme.demo.dto.UploadWordFileResponseDto;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
|
||||
|
||||
|
||||
@Service
|
||||
public class KnowledgeIngestService {
|
||||
|
||||
@Value("${humbee.login.email}")
|
||||
private String email;
|
||||
|
||||
@Value("${humbee.login.password}")
|
||||
private String password;
|
||||
|
||||
@Value("${openai.enabled}")
|
||||
private boolean openAiEnabled;
|
||||
|
||||
private final WebClient webClient;
|
||||
|
||||
private final MultiValueMap<String, String> sessionCookies = new LinkedMultiValueMap<>();
|
||||
|
||||
@Autowired
|
||||
private DocumentStorageService documentStorageService;
|
||||
|
||||
@Autowired
|
||||
private PromptService promptService;
|
||||
|
||||
public KnowledgeIngestService () {
|
||||
|
||||
|
||||
ExchangeStrategies strategies = ExchangeStrategies.builder()
|
||||
.codecs(configurer -> configurer.defaultCodecs().maxInMemorySize(10 * 1024 * 1024))
|
||||
.build();
|
||||
|
||||
this.webClient = WebClient.builder()
|
||||
.baseUrl("https://cloud.humbee.de")
|
||||
.exchangeStrategies(strategies)
|
||||
.build();
|
||||
|
||||
}
|
||||
public List<ApiResponse<FileProcessedDto>> buildKnowledgeBase() throws Exception{
|
||||
|
||||
login();
|
||||
String json = getDiroctoryJson();
|
||||
List<String> links = extractWordLinks(json);
|
||||
|
||||
ExecutorService pool = Executors.newFixedThreadPool(6);
|
||||
try {
|
||||
|
||||
List<Future<ApiResponse<FileProcessedDto>>> futures = new ArrayList<>();
|
||||
|
||||
for(String link :links) {
|
||||
|
||||
futures.add(pool.submit(() -> processOneFile(link)));
|
||||
|
||||
}
|
||||
List<ApiResponse<FileProcessedDto>> result = new ArrayList<>();
|
||||
|
||||
for(Future<ApiResponse<FileProcessedDto>> f: futures) {
|
||||
|
||||
result.add(f.get());
|
||||
}
|
||||
return result;
|
||||
|
||||
|
||||
} catch (InterruptedException | ExecutionException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new RuntimeException("Fehler bei paralleler Dateiverarbeitung: " + e.getMessage());
|
||||
} finally {
|
||||
|
||||
pool.shutdown();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
public ApiResponse<FileProcessedDto> processOneFile(String link){
|
||||
|
||||
|
||||
try {
|
||||
String rawText = readWordFile(link);
|
||||
|
||||
|
||||
String cleanText = openAiEnabled
|
||||
? promptService.removeSpeakerLabelsAndTimestamps(rawText)
|
||||
: rawText;
|
||||
|
||||
|
||||
ResponseEntity<String> saveResponse = documentStorageService.saveDocuemntWithChuncks(link, rawText, cleanText);
|
||||
|
||||
|
||||
String statusResponse = saveResponse.getStatusCode().toString();
|
||||
FileProcessedDto fileProcessedDto = FileProcessedDto.builder()
|
||||
.link(link)
|
||||
.status(statusResponse)
|
||||
.build();
|
||||
|
||||
return ApiResponse.ok(fileProcessedDto);
|
||||
|
||||
|
||||
}catch (Exception e ) {
|
||||
|
||||
return ApiResponse.error("Fehler bei "+ link + ": "+ e.getMessage());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public void login() {
|
||||
|
||||
String loginPath = "/account/login";
|
||||
sessionCookies.clear();
|
||||
Map<String, String> requestBody = new HashMap<>();
|
||||
requestBody.put("email", email);
|
||||
requestBody.put("password", password);
|
||||
|
||||
ClientResponse response = webClient.post()
|
||||
.uri(loginPath)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.accept(MediaType.ALL)
|
||||
.bodyValue(requestBody)
|
||||
.exchangeToMono(res -> Mono.just(res))
|
||||
.block();
|
||||
|
||||
if(response == null) {
|
||||
throw new RuntimeException("Anmeldung fehlgeschlagen: Keine Antwort vom Server.");
|
||||
}
|
||||
|
||||
if(!(response.statusCode().is2xxSuccessful() || response.statusCode().is3xxRedirection())) {
|
||||
throw new RuntimeException("Anmeldung fehlgeschlagen. Status: " + response.statusCode().value());
|
||||
}
|
||||
|
||||
Map<String, List<ResponseCookie>> cookiesMap = response.cookies();
|
||||
|
||||
for(String name : cookiesMap.keySet()) {
|
||||
|
||||
List<ResponseCookie> values = cookiesMap.get(name);
|
||||
|
||||
for(ResponseCookie cookie : values) {
|
||||
|
||||
sessionCookies.add(name, cookie.getValue());
|
||||
}
|
||||
}
|
||||
if(sessionCookies.isEmpty()) {
|
||||
throw new RuntimeException("Anmeldung fehlgeschlagen: Keine Cookies vom Server erhalten.");
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
public String getDiroctoryJson() {
|
||||
|
||||
ensureLoggedIn();
|
||||
String responseBody = webClient.get()
|
||||
.uri("/directory/a18eb2ff-d045-4926-8707-322be2588aab/content")
|
||||
.accept(MediaType.APPLICATION_JSON)
|
||||
.cookies(cookies -> cookies.addAll(sessionCookies))
|
||||
.retrieve()
|
||||
.bodyToMono(String.class)
|
||||
.block();
|
||||
|
||||
if(responseBody == null || responseBody.isEmpty()) {
|
||||
throw new RuntimeException("Das Verzeichnis konnte nicht geladen werden oder ist leer.");
|
||||
}
|
||||
|
||||
|
||||
return responseBody;
|
||||
|
||||
|
||||
|
||||
}
|
||||
public List<String> extractWordLinks(String json) throws Exception{
|
||||
|
||||
List<String> links = new ArrayList<>();
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
JsonNode root = objectMapper.readTree(json);
|
||||
JsonNode items = root.get("items");
|
||||
if(items == null || !items.isArray()) {
|
||||
return links;
|
||||
}
|
||||
|
||||
for(JsonNode item:items) {
|
||||
|
||||
JsonNode quickActions = item.path("_actions").path("quickActions");
|
||||
|
||||
for(JsonNode action: quickActions) {
|
||||
String id = action.path("id").asText();
|
||||
String href = action.path("href").asText();
|
||||
|
||||
if("download".equals(id) && (href.contains(".docx?") || href.contains(".doc?") || href.contains(".txt"))) {
|
||||
links.add("https://cloud.humbee.de"+href);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return links;
|
||||
}
|
||||
|
||||
|
||||
public String readWordFile(String downloadHref) throws Exception{
|
||||
|
||||
ensureLoggedIn();
|
||||
if(downloadHref == null || downloadHref.isBlank()) {
|
||||
throw new RuntimeException("Der Download-Link ist leer oder ungültig.");
|
||||
}
|
||||
byte[] fileBytes = webClient.get()
|
||||
.uri(downloadHref)
|
||||
.cookies(cookies -> cookies.addAll(sessionCookies))
|
||||
.retrieve()
|
||||
.bodyToMono(byte[].class)
|
||||
.block();
|
||||
|
||||
|
||||
if(downloadHref.contains(".docx")) {
|
||||
try(XWPFDocument document = new XWPFDocument(new ByteArrayInputStream(fileBytes))) {
|
||||
|
||||
StringBuilder text = new StringBuilder();
|
||||
for(XWPFParagraph paragraph : document.getParagraphs()) {
|
||||
text.append(paragraph.getText()).append("\n");
|
||||
|
||||
}
|
||||
|
||||
return text.toString();
|
||||
|
||||
}catch(Exception e) {
|
||||
throw new RuntimeException("Die DDC-Datei konnte nicht gelesen werden.");
|
||||
}
|
||||
|
||||
}else if(downloadHref.contains(".doc")) {
|
||||
try(HWPFDocument document = new HWPFDocument(new ByteArrayInputStream(fileBytes));
|
||||
WordExtractor extractor = new WordExtractor(document)){
|
||||
return extractor.getText();
|
||||
|
||||
}
|
||||
}
|
||||
else if(downloadHref.contains(".txt")){
|
||||
|
||||
return decodeBytes(fileBytes);
|
||||
|
||||
}
|
||||
throw new RuntimeException("Nicht unterstützer Dateityp");
|
||||
|
||||
}
|
||||
|
||||
private void ensureLoggedIn() {
|
||||
if(sessionCookies.isEmpty()) {
|
||||
throw new RuntimeException("Nicht angeledet. Bitte zuerst einloggen.");
|
||||
}
|
||||
}
|
||||
|
||||
public String decodeBytes(byte[] bytes) {
|
||||
if (bytes == null || bytes.length == 0) return "";
|
||||
|
||||
|
||||
if (bytes.length >= 3 && (bytes[0]&0xFF)==0xEF && (bytes[1]&0xFF)==0xBB && (bytes[2]&0xFF)==0xBF)
|
||||
return new String(bytes, 3, bytes.length-3, java.nio.charset.StandardCharsets.UTF_8);
|
||||
if (bytes.length >= 2 && (bytes[0]&0xFF)==0xFF && (bytes[1]&0xFF)==0xFE)
|
||||
return new String(bytes, 2, bytes.length-2, java.nio.charset.StandardCharsets.UTF_16LE);
|
||||
if (bytes.length >= 2 && (bytes[0]&0xFF)==0xFE && (bytes[1]&0xFF)==0xFF)
|
||||
return new String(bytes, 2, bytes.length-2, java.nio.charset.StandardCharsets.UTF_16BE);
|
||||
|
||||
int nulls = 0;
|
||||
for (int i = 1; i < Math.min(bytes.length, 400); i += 2)
|
||||
if (bytes[i] == 0) nulls++;
|
||||
if (nulls > 50)
|
||||
return new String(bytes, java.nio.charset.StandardCharsets.UTF_16LE);
|
||||
|
||||
return new String(bytes, java.nio.charset.StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
public UploadWordFileResponseDto uploadWordFile(String fileName, byte[] docxBytes) throws Exception{
|
||||
|
||||
login();
|
||||
|
||||
MultipartBodyBuilder builder = new MultipartBodyBuilder();
|
||||
builder.part("file", new ByteArrayResource(docxBytes))
|
||||
.filename(fileName)
|
||||
.contentType(MediaType.parseMediaType(
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document"));
|
||||
|
||||
String response = webClient.post()
|
||||
.uri("https://cloud.humbee.de/directory/a18eb2ff-d045-4926-8707-322be2588aab/content")
|
||||
.cookies(c -> c.addAll(sessionCookies))
|
||||
.contentType(MediaType.MULTIPART_FORM_DATA)
|
||||
.body(BodyInserters.fromMultipartData(builder.build()))
|
||||
.retrieve()
|
||||
.bodyToMono(String.class)
|
||||
.block();
|
||||
|
||||
System.out.println(" Humbee Upload Antwort: "+ response);
|
||||
|
||||
String json = getDiroctoryJson();
|
||||
List<String> links = extractWordLinks(json);
|
||||
|
||||
for(String link: links) {
|
||||
|
||||
if(link.contains(fileName)) {
|
||||
|
||||
return UploadWordFileResponseDto.builder()
|
||||
.success(true)
|
||||
.fileName(fileName)
|
||||
.link(link)
|
||||
.message("Word-Datei erfolgreichhochgeladen")
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
return UploadWordFileResponseDto.builder()
|
||||
.success(false)
|
||||
.fileName(fileName)
|
||||
.link(null)
|
||||
.message("Hochgeladen, aber im Verzeichnis nicht gefunden")
|
||||
.build();
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user