diff --git a/.DS_Store b/.DS_Store index 0841c84..8b7ccd0 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.env.example b/.env.example index 1f45a2e..ff21fcd 100644 --- a/.env.example +++ b/.env.example @@ -16,3 +16,6 @@ HUMBEE_PASSWORD= # Optional extra JVM flags (e.g. -Xmx512m) JAVA_TOOL_OPTIONS= + +# --- Admin-Token: schuetzt die Freigabe-Endpunkte (/contributions/pending, /approve) --- +HOMME_ADMIN_TOKEN= diff --git a/HOEMMA_AENDERUNGSPLAN.md b/HOEMMA_AENDERUNGSPLAN.md new file mode 100644 index 0000000..b4848f9 --- /dev/null +++ b/HOEMMA_AENDERUNGSPLAN.md @@ -0,0 +1,273 @@ +# HÖMMA – Änderungsplan (Backend an Plugin anpassen) + +خطة التعديل لمطابقة الباك-إند مع إضافة **HÖMMA Human Library**. +الفكرة: نضيف **Controller جديد واحد** يوفّر النقاط الثلاث تماماً كما يتوقّعها الـ plugin، +ويستدعي الخدمات الموجودة أصلاً. لا نلمس الـ Controllers الشغّالة. + +الطريقة المختارة: **تعديل الباك-إند** (وليس الـ plugin). +النقاط الثلاث النهائية التي ستضعها في ووردبريس: + +| خانة ووردبريس | الرابط النهائي (مثال Azure) | +|---|---| +| Speichern-Endpunkt | `https:///human-library` | +| Transkriptions-Endpunkt | `https:///transcribe` | +| Extraktions-Endpunkt | `https:///extract` | + +--- + +## 1) pom.xml — إضافة دعم PDF + +الـ plugin يسمح برفع Word **و PDF**، لكن المشروع يعالج `.docx` فقط. +أضف مكتبة PDFBox داخل ``: + +```xml + + org.apache.pdfbox + pdfbox + 2.0.31 + +``` + +--- + +## 2) WordExportService.java — تفعيل قراءة PDF + +الملف: `src/main/java/com/homme/demo/service/WordExportService.java` + +داخل دالة `leseDatei(MultipartFile file)`، **قبل** سطر `return knowledgeIngestService.decodeBytes(bytes);` +أضف فرع PDF: + +```java +if (name.endsWith(".pdf")) { + try (org.apache.pdfbox.pdmodel.PDDocument doc = + org.apache.pdfbox.pdmodel.PDDocument.load(bytes)) { + return new org.apache.pdfbox.text.PDFTextStripper().getText(doc); + } +} +``` + +بذلك: `.docx` عبر POI، `.pdf` عبر PDFBox، والباقي يبقى كما هو. + +--- + +## 3) DTO جديد — طلب الحفظ (JSON) + +ملف جديد: `src/main/java/com/homme/demo/dto/HumanLibraryRequest.java` + +الـ plugin يرسل JSON بهذه الحقول: `source, name, gender, city, text, consent`. + +```java +package com.homme.demo.dto; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class HumanLibraryRequest { + private String source; + private String name; + private String gender; + private String city; + private String text; + private boolean consent; +} +``` + +--- + +## 4) DTO جديد — الرد الموحّد `{ "text": "..." }` + +ملف جديد: `src/main/java/com/homme/demo/dto/TextResponse.java` + +الـ plugin يتوقّع من الاستخراج والتفريغ رداً بصيغة `{"text":"..."}`. + +```java +package com.homme.demo.dto; + +public record TextResponse(String text) { +} +``` + +--- + +## 5) Controller جديد — النقاط الثلاث للـ plugin + +ملف جديد: `src/main/java/com/homme/demo/controller/HumanLibraryController.java` + +هذا هو قلب التعديل. يوفّر `/extract`, `/transcribe`, `/human-library` +بنفس أسماء الحقول والصيغ التي يتوقّعها الـ plugin، ويستدعي الخدمات الموجودة. + +```java +package com.homme.demo.controller; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import com.homme.demo.dto.HumanLibraryRequest; +import com.homme.demo.dto.TextResponse; +import com.homme.demo.dto.UploadWordFileResponseDto; +import com.homme.demo.service.AzureSpeechService; +import com.homme.demo.service.KnowledgeIngestService; +import com.homme.demo.service.WordExportService; + +import java.time.LocalDate; + +@RestController +public class HumanLibraryController { + + @Autowired private WordExportService wordExportService; + @Autowired private AzureSpeechService azureSpeechService; + @Autowired private KnowledgeIngestService knowledgeIngestService; + + // (1) Extraktion: Word/PDF -> { "text": "..." } + @PostMapping(value = "/extract", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public ResponseEntity extract(@RequestParam("file") MultipartFile file) throws Exception { + if (file == null || file.isEmpty()) { + return ResponseEntity.badRequest().body(new TextResponse("")); + } + String text = wordExportService.leseDatei(file); + return ResponseEntity.ok(new TextResponse(text == null ? "" : text)); + } + + // (2) Transkription: audio -> { "text": "..." } + @PostMapping(value = "/transcribe", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public ResponseEntity transcribe(@RequestParam("audio") MultipartFile audio) throws Exception { + if (audio == null || audio.isEmpty()) { + return ResponseEntity.badRequest().body(new TextResponse("")); + } + String text = azureSpeechService.transcribe(audio.getBytes(), audio.getOriginalFilename()); + return ResponseEntity.ok(new TextResponse(text == null ? "" : text)); + } + + // (3) Speichern: JSON { source, name, gender, city, text, consent } -> 201 + @PostMapping(value = "/human-library", consumes = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity saveContribution(@RequestBody HumanLibraryRequest req) { + try { + if (!req.isConsent()) { + return ResponseEntity.badRequest().body("Einwilligung (consent) erforderlich."); + } + if (req.getText() == null || req.getText().isBlank()) { + return ResponseEntity.badRequest().body("Text erforderlich."); + } + + String title = (req.getName() == null || req.getName().isBlank()) + ? "Anonymous" : req.getName().trim().replace(" ", "_"); + String city = (req.getCity() == null || req.getCity().isBlank()) + ? "Dortmund" : req.getCity(); + + // Text -> DOCX -> Humbee-Upload -> RAG-Verarbeitung (wie save-new-interview) + byte[] docx = wordExportService.createInterviewDocx(req.getText()); + String fileName = "%s_%s_%s.docx".formatted(LocalDate.now(), title, city); + + UploadWordFileResponseDto upload = knowledgeIngestService.uploadWordFile(fileName, docx); + if (!upload.isSuccess()) { + return ResponseEntity.status(HttpStatus.BAD_GATEWAY) + .body("Upload fehlgeschlagen: " + upload.getMessage()); + } + + knowledgeIngestService.processOneFile(upload.getLink()); + return ResponseEntity.status(HttpStatus.CREATED).body("{\"status\":\"ok\"}"); + + } catch (Exception e) { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body("Fehler beim Speichern: " + e.getMessage()); + } + } +} +``` + +> ملاحظة: هذا يعيد استخدام نفس منطق `save-new-interview` الموجود. +> إذا أردت لاحقاً مرحلة **مراجعة تحريرية (ausstehend)** قبل RAG، نضيف حالة/جدول قبل استدعاء `processOneFile`. +> حقلا `gender` و `source` يُستقبلان لكن لا يُخزَّنان حالياً — أضِفهما للـ entity لاحقاً إن لزم. + +--- + +## 6) CORS — (اختياري، موصى به) + +الملف: `src/main/java/com/homme/demo/config/WebConfig.java` +حالياً مفتوح للكل (`allowedOriginPatterns("*")`) — يعمل. +للحصر على نطاقك فقط (كما نصح صاحب العمل) استبدله بـ: + +```java +registry.addMapping("/**") + .allowedOrigins("https://hmmahumanlibraryruhr1314.live-website.com") + .allowedMethods("GET", "POST", "OPTIONS") + .allowedHeaders("*") + .allowCredentials(false) + .maxAge(3600); +``` + +(اترك `*` إن أردت تبسيط الاختبار الآن.) + +--- + +## 7) Dockerfile — جديد في جذر المشروع + +ملف جديد: `Dockerfile` (بجانب `pom.xml`) + +```dockerfile +FROM maven:3.9-eclipse-temurin-17 AS build +WORKDIR /app +COPY pom.xml . +RUN mvn -q dependency:go-offline +COPY src ./src +RUN mvn -q clean package -DskipTests + +FROM eclipse-temurin:17-jre +WORKDIR /app +COPY --from=build /app/target/*.jar app.jar +EXPOSE 9192 +ENTRYPOINT ["java", "-jar", "app.jar"] +``` + +ملف جديد: `.dockerignore` + +``` +target/ +.git/ +.settings/ +.classpath +.project +*.iml +``` + +--- + +## 8) متغيّرات البيئة (Azure) — ليست تعديل كود + +عند الرفع على Azure App Service اضبط: + +``` +WEBSITES_PORT=9192 +DB_PASSWORD=... +OPENAI_API_MISTRAL_KEY=... +AZURE_EMBEDDING_KEY=... +AZURE_SPEECH_KEY=... +HUMBEE_PASSWORD=... +``` + +وفعّل **Managed Identity** للتطبيق (بسبب `DefaultAzureCredential`)، +وامنحها صلاحية على مورد `homme-foundry-dev`، +واسمح لـ PostgreSQL بالاتصال (Allow Azure services). + +--- + +## ملخّص الملفات + +| # | الملف | النوع | +|---|---|---| +| 1 | `pom.xml` | تعديل (dependency) | +| 2 | `service/WordExportService.java` | تعديل (فرع PDF) | +| 3 | `dto/HumanLibraryRequest.java` | جديد | +| 4 | `dto/TextResponse.java` | جديد | +| 5 | `controller/HumanLibraryController.java` | جديد | +| 6 | `config/WebConfig.java` | تعديل (اختياري) | +| 7 | `Dockerfile` + `.dockerignore` | جديد | +| 8 | متغيّرات Azure | إعداد (لا كود) | + +بعد هذه التعديلات: تبني بـ Docker → ترفع على Azure → تضع الروابط الثلاثة في صفحة إعدادات HÖMMA Human Library → يبدأ النموذج يعمل بالكامل (استخراج، تفريغ، حفظ). diff --git a/pom.xml b/pom.xml index b58bc69..0685091 100644 --- a/pom.xml +++ b/pom.xml @@ -81,6 +81,12 @@ org.springframework.boot spring-boot-starter-data-jpa + + + org.apache.pdfbox + pdfbox + 2.0.31 + org.apache.poi diff --git a/src/.DS_Store b/src/.DS_Store index 91e64fa..a576afa 100644 Binary files a/src/.DS_Store and b/src/.DS_Store differ diff --git a/src/main/.DS_Store b/src/main/.DS_Store index 383e1bd..f3a55bc 100644 Binary files a/src/main/.DS_Store and b/src/main/.DS_Store differ diff --git a/src/main/java/com/homme/demo/config/RateLimitFilter.java b/src/main/java/com/homme/demo/config/RateLimitFilter.java new file mode 100644 index 0000000..20eea83 --- /dev/null +++ b/src/main/java/com/homme/demo/config/RateLimitFilter.java @@ -0,0 +1,51 @@ +package com.homme.demo.config; + +import java.io.IOException; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicInteger; + +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +@Component +public class RateLimitFilter extends OncePerRequestFilter { + + private static final int MAX_REQUESTS_PER_MINUTE = 20; + + private final Cache counters = Caffeine.newBuilder() + .expireAfterWrite(Duration.ofMinutes(1)) + .build(); + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) + throws ServletException, IOException { + + String path = request.getRequestURI(); + + boolean isProtected = path.startsWith("/knowledge-base/extract") + || path.startsWith("/knowledge-base/transcription") + || path.startsWith("/knowledge-base/contributions") + || path.startsWith("/chat/"); + + if (isProtected) { + String ip = request.getRemoteAddr(); + AtomicInteger count = counters.get(ip, k -> new AtomicInteger(0)); + if (count.incrementAndGet() > MAX_REQUESTS_PER_MINUTE) { + response.setStatus(429); + response.setContentType("application/json"); + response.getWriter().write("{\"error\": \"Zu viele Anfragen. Bitte später erneut versuchen.\"}"); + return; + } + } + + chain.doFilter(request, response); + } +} diff --git a/src/main/java/com/homme/demo/config/WebConfig.java b/src/main/java/com/homme/demo/config/WebConfig.java index a0cfd75..d2e0182 100644 --- a/src/main/java/com/homme/demo/config/WebConfig.java +++ b/src/main/java/com/homme/demo/config/WebConfig.java @@ -1,5 +1,6 @@ package com.homme.demo.config; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Configuration; import org.springframework.web.servlet.config.annotation.CorsRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; @@ -8,10 +9,13 @@ import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; @Configuration public class WebConfig implements WebMvcConfigurer { + @Value("${homme.cors.allowed-origins}") + private String[] allowedOrigins; + @Override public void addCorsMappings(CorsRegistry registry) { registry.addMapping("/**") - .allowedOriginPatterns("*") + .allowedOrigins(allowedOrigins) .allowedMethods("GET", "POST", "OPTIONS") .allowedHeaders("*") .allowCredentials(false) diff --git a/src/main/java/com/homme/demo/controller/ChatController.java b/src/main/java/com/homme/demo/controller/ChatController.java index e9e1474..42e370a 100644 --- a/src/main/java/com/homme/demo/controller/ChatController.java +++ b/src/main/java/com/homme/demo/controller/ChatController.java @@ -1,7 +1,6 @@ package com.homme.demo.controller; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.CrossOrigin; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -14,7 +13,6 @@ import com.homme.demo.dto.ChatRequest; import com.homme.demo.service.RagService; -@CrossOrigin(origins = "${homme.cors.allowed-origins}") @RestController @RequestMapping("/chat") public class ChatController { diff --git a/src/main/java/com/homme/demo/controller/IngestController.java b/src/main/java/com/homme/demo/controller/IngestController.java index 5238b1b..d775283 100644 --- a/src/main/java/com/homme/demo/controller/IngestController.java +++ b/src/main/java/com/homme/demo/controller/IngestController.java @@ -1,10 +1,14 @@ package com.homme.demo.controller; +import java.util.Map; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; import com.homme.demo.service.WordPressIngestService; @@ -25,4 +29,5 @@ public class IngestController { } + } diff --git a/src/main/java/com/homme/demo/controller/KnowledgeIngestController.java b/src/main/java/com/homme/demo/controller/KnowledgeIngestController.java index 3ea59fd..a9178db 100644 --- a/src/main/java/com/homme/demo/controller/KnowledgeIngestController.java +++ b/src/main/java/com/homme/demo/controller/KnowledgeIngestController.java @@ -2,25 +2,36 @@ package com.homme.demo.controller; import java.io.IOException; import java.time.LocalDate; +import java.time.LocalDateTime; import java.util.List; +import java.util.Map; +import java.util.Set; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.ModelAttribute; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.server.ResponseStatusException; import com.homme.demo.dto.ApiResponse; +import com.homme.demo.dto.ContributionRequest; import com.homme.demo.dto.FileProcessedDto; import com.homme.demo.dto.UploadWordFileResponseDto; +import com.homme.demo.entity.Contribution; import com.homme.demo.input.ContentInput; +import com.homme.demo.repository.ContributionRepository; import com.homme.demo.service.AzureSpeechService; import com.homme.demo.service.KnowledgeIngestService; import com.homme.demo.service.WordExportService; @@ -30,46 +41,154 @@ import com.homme.demo.service.WordExportService; @RequestMapping("/knowledge-base") public class KnowledgeIngestController { + private static final Set ALLOWED_DOC_EXT = Set.of("docx", "doc", "pdf", "txt"); + private static final Set ALLOWED_AUDIO_EXT = Set.of("webm", "ogg", "mp3", "wav", "m4a", "mp4"); + @Autowired private WordExportService wordExportService; @Autowired private KnowledgeIngestService knowledgeIngestService; - - @Autowired - private AzureSpeechService azureSpeechService; - + @Autowired + private AzureSpeechService azureSpeechService; + + @Autowired + private ContributionRepository contributionRepository; + + @Value("${homme.admin.token}") + private String adminToken; + + @GetMapping("/build") public List> buildKnowledgeBase() throws Exception { return knowledgeIngestService.buildKnowledgeBase(); } - - @PostMapping("/transcription") - public String transcribe(@RequestParam("file") MultipartFile file) throws IOException{ - - - - byte[] audioBytes = file.getBytes(); - String fileName = file.getOriginalFilename(); - return azureSpeechService.transcribe(audioBytes, fileName); - + + @PostMapping("/transcription") + public ResponseEntity> transcribe(@RequestParam("audio") MultipartFile audio) throws IOException { + + String ext = getExtension(audio.getOriginalFilename()); + if (!ALLOWED_AUDIO_EXT.contains(ext)) { + return new ResponseEntity<>(Map.of("error", "Nicht unterstütztes Audioformat."), HttpStatus.BAD_REQUEST); + } + + String text = azureSpeechService.transcribe(audio.getBytes(), audio.getOriginalFilename()); + return ResponseEntity.ok(Map.of("text", text)); } - + + + @PostMapping("/extract") + public ResponseEntity> extract(@RequestParam("file") MultipartFile file) throws Exception { + + String ext = getExtension(file.getOriginalFilename()); + if (!ALLOWED_DOC_EXT.contains(ext)) { + return new ResponseEntity<>(Map.of("error", "Nur DOCX, DOC, PDF oder TXT erlaubt."), HttpStatus.BAD_REQUEST); + } + + String text = knowledgeIngestService.extractText(file.getBytes(), file.getOriginalFilename()); + return ResponseEntity.ok(Map.of("text", text)); + } + + + @PostMapping(value = "/contributions", consumes = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity> saveContribution(@RequestBody ContributionRequest request) { + + if (request.getConsent() == null || !request.getConsent()) { + return new ResponseEntity<>(Map.of("error", "Einwilligung ist erforderlich."), HttpStatus.BAD_REQUEST); + } + if (request.getName() == null || request.getName().isBlank()) { + return new ResponseEntity<>(Map.of("error", "Name ist erforderlich."), HttpStatus.BAD_REQUEST); + } + if (request.getText() == null || request.getText().isBlank()) { + return new ResponseEntity<>(Map.of("error", "Text darf nicht leer sein."), HttpStatus.BAD_REQUEST); + } + + Contribution contribution = Contribution.builder() + .source(request.getSource()) + .name(request.getName().trim()) + .gender(request.getGender()) + .city((request.getCity() == null || request.getCity().isBlank()) ? "Dortmund" : request.getCity().trim()) + .text(request.getText()) + .consent(true) + .status("PENDING") + .createdAt(LocalDateTime.now()) + .build(); + + Contribution saved = contributionRepository.save(contribution); + + return new ResponseEntity<>(Map.of("id", saved.getId(), "status", saved.getStatus()), HttpStatus.CREATED); + } + + + @GetMapping("/contributions/pending") + public List pendingContributions(@RequestHeader("X-Admin-Token") String token) { + checkAdminToken(token); + return contributionRepository.findByStatus("PENDING"); + } + + + @PostMapping("/contributions/{id}/approve") + public ResponseEntity> approveContribution(@PathVariable Long id, + @RequestHeader("X-Admin-Token") String token) { + + checkAdminToken(token); + try { + Contribution contribution = contributionRepository.findById(id) + .orElseThrow(() -> new RuntimeException("Beitrag nicht gefunden.")); + + byte[] docx = wordExportService.createInterviewDocx(contribution.getText()); + + String fileName = "%s_%s_%s.docx".formatted( + LocalDate.now(), + contribution.getName().replace(" ", "_"), + contribution.getCity()); + + UploadWordFileResponseDto upload = knowledgeIngestService.uploadWordFile(fileName, docx); + if (!upload.isSuccess()) { + return new ResponseEntity<>(ApiResponse.error("Upload fehlgeschlagen: " + upload.getMessage()), + HttpStatus.BAD_REQUEST); + } + + ApiResponse processed = knowledgeIngestService.processOneFile(upload.getLink()); + + contribution.setStatus("APPROVED"); + contributionRepository.save(contribution); + + return new ResponseEntity<>(processed, HttpStatus.OK); + + } catch (Exception e) { + return new ResponseEntity<>(ApiResponse.error("Fehler: " + e.getMessage()), + HttpStatus.INTERNAL_SERVER_ERROR); + } + } + + + private void checkAdminToken(String token) { + if (adminToken == null || adminToken.isBlank() || !adminToken.equals(token)) { + throw new ResponseStatusException(HttpStatus.UNAUTHORIZED, "Ungültiger Admin-Token"); + } + } + + + private String getExtension(String fileName) { + if (fileName == null || !fileName.contains(".")) return ""; + return fileName.substring(fileName.lastIndexOf('.') + 1).toLowerCase(); + } + + @PostMapping(value = "/save-new-interview", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) public ResponseEntity> saveNewInterview(@ModelAttribute ContentInput contentInput ) { - - System.out.println("rrrr "); try { String rawText; - + if (contentInput.getFile() != null && !contentInput.getFile().isEmpty()) { rawText = wordExportService.leseDatei(contentInput.getFile()); - + } else if (contentInput.getText() != null && !contentInput.getText().isBlank()) { rawText = contentInput.getText(); @@ -81,28 +200,28 @@ public class KnowledgeIngestController { return new ResponseEntity<>(ApiResponse.error("Kein lesbarer Inhalt gefunden."), HttpStatus.BAD_REQUEST); } - + String title = (contentInput.getTitle() == null || contentInput.getTitle().isBlank()) ? "Anonymous" : contentInput.getTitle(); - + String city = (contentInput.getCity() == null|| contentInput.getCity().isBlank()) ? "Dortmund" : contentInput.getCity(); - - - + + + byte[] docx = wordExportService.createInterviewDocx(rawText); - + String fileName = "%s_%s_%s.docx".formatted(LocalDate.now(),title.trim().replace(" ", "_"),city); - + UploadWordFileResponseDto upload = knowledgeIngestService.uploadWordFile(fileName, docx); - + if(!upload.isSuccess()) { - + return new ResponseEntity<>(ApiResponse.error("Upload fehlgeschlagen: " +upload.getMessage()), HttpStatus.BAD_REQUEST); - + } - + ApiResponse processed = knowledgeIngestService.processOneFile(upload.getLink()); return new ResponseEntity<>(processed, HttpStatus.CREATED); @@ -112,7 +231,3 @@ public class KnowledgeIngestController { } } } - - - - diff --git a/src/main/java/com/homme/demo/dto/ContributionRequest.java b/src/main/java/com/homme/demo/dto/ContributionRequest.java new file mode 100644 index 0000000..82880e4 --- /dev/null +++ b/src/main/java/com/homme/demo/dto/ContributionRequest.java @@ -0,0 +1,19 @@ +package com.homme.demo.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@JsonIgnoreProperties(ignoreUnknown = true) +public class ContributionRequest { + + private String source; + private String name; + private String gender; + private String city; + private String text; + private Boolean consent; +} diff --git a/src/main/java/com/homme/demo/entity/Contribution.java b/src/main/java/com/homme/demo/entity/Contribution.java new file mode 100644 index 0000000..6df24be --- /dev/null +++ b/src/main/java/com/homme/demo/entity/Contribution.java @@ -0,0 +1,44 @@ +package com.homme.demo.entity; + +import java.time.LocalDateTime; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Entity +@Setter +@Getter +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class Contribution { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + private String source; + + private String name; + + private String gender; + + private String city; + + @Column(columnDefinition = "TEXT") + private String text; + + private boolean consent; + + private String status; // PENDING, APPROVED, REJECTED + + private LocalDateTime createdAt; +} diff --git a/src/main/java/com/homme/demo/repository/ContributionRepository.java b/src/main/java/com/homme/demo/repository/ContributionRepository.java new file mode 100644 index 0000000..65b3c21 --- /dev/null +++ b/src/main/java/com/homme/demo/repository/ContributionRepository.java @@ -0,0 +1,12 @@ +package com.homme.demo.repository; + +import java.util.List; + +import org.springframework.data.jpa.repository.JpaRepository; + +import com.homme.demo.entity.Contribution; + +public interface ContributionRepository extends JpaRepository { + + List findByStatus(String status); +} diff --git a/src/main/java/com/homme/demo/service/KnowledgeIngestService.java b/src/main/java/com/homme/demo/service/KnowledgeIngestService.java index 544d0f5..ad5cdd7 100644 --- a/src/main/java/com/homme/demo/service/KnowledgeIngestService.java +++ b/src/main/java/com/homme/demo/service/KnowledgeIngestService.java @@ -13,7 +13,8 @@ import java.util.concurrent.ExecutionException; import org.apache.poi.xwpf.usermodel.XWPFDocument; import org.apache.poi.xwpf.usermodel.XWPFParagraph; - +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.text.PDFTextStripper; import org.apache.poi.hwpf.HWPFDocument; import org.apache.poi.hwpf.extractor.WordExtractor; import org.springframework.beans.factory.annotation.Autowired; @@ -248,9 +249,15 @@ public class KnowledgeIngestService { .retrieve() .bodyToMono(byte[].class) .block(); - - if(downloadHref.contains(".docx")) { + return extractText(fileBytes, downloadHref); + } + + public String extractText(byte[] fileBytes, String fileName) throws Exception{ + + String name = (fileName == null) ? "" : fileName.toLowerCase(); + + if(name.contains(".docx")) { try(XWPFDocument document = new XWPFDocument(new ByteArrayInputStream(fileBytes))) { StringBuilder text = new StringBuilder(); @@ -265,14 +272,22 @@ public class KnowledgeIngestService { throw new RuntimeException("Die DDC-Datei konnte nicht gelesen werden."); } - }else if(downloadHref.contains(".doc")) { + }else if(name.contains(".doc")) { try(HWPFDocument document = new HWPFDocument(new ByteArrayInputStream(fileBytes)); WordExtractor extractor = new WordExtractor(document)){ return extractor.getText(); } } - else if(downloadHref.contains(".txt")){ + + else if(name.endsWith(".pdf")) { + + try(PDDocument doc = PDDocument.load(fileBytes)){ + String rawText= new PDFTextStripper().getText(doc); + return rawText; + } + } + else if(name.contains(".txt")){ return decodeBytes(fileBytes); @@ -293,8 +308,10 @@ public class KnowledgeIngestService { if (bytes.length >= 3 && (bytes[0]&0xFF)==0xEF && (bytes[1]&0xFF)==0xBB && (bytes[2]&0xFF)==0xBF) return new String(bytes, 3, bytes.length-3, java.nio.charset.StandardCharsets.UTF_8); + if (bytes.length >= 2 && (bytes[0]&0xFF)==0xFF && (bytes[1]&0xFF)==0xFE) return new String(bytes, 2, bytes.length-2, java.nio.charset.StandardCharsets.UTF_16LE); + if (bytes.length >= 2 && (bytes[0]&0xFF)==0xFE && (bytes[1]&0xFF)==0xFF) return new String(bytes, 2, bytes.length-2, java.nio.charset.StandardCharsets.UTF_16BE); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 024eb54..c174093 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -8,7 +8,10 @@ spring.config.import=optional:classpath:/.env[.properties] server.port =9192 #Origins -homme.cors.allowed-origins=https://hmmahumanlibraryruhr1314.live-website.com +homme.cors.allowed-origins=http://localhost:9192,https://hmmahumanlibraryruhr1314.live-website.com,https://xn--hmma-5qa.org,https://www.xn--hmma-5qa.org + +#Admin-Token fuer Freigabe von Beitraegen (Wert kommt aus .env) +homme.admin.token=${HOMME_ADMIN_TOKEN} spring.datasource.url=jdbc:postgresql://homme-pg-dev-01.postgres.database.azure.com:5432/postgres?sslmode=require spring.datasource.username=mohammadadmin @@ -56,6 +59,6 @@ logging.level.org.springframework.security=DEBUG humbee.login.email=mohammad.zwaib@borsig11.de humbee.login.password=${HUMBEE_PASSWORD} -#Upload-Limits (Audio-Interviews koennen gross sein) -spring.servlet.multipart.max-file-size=300MB -spring.servlet.multipart.max-request-size=300MB \ No newline at end of file +#Upload-Limits +spring.servlet.multipart.max-file-size=25MB +spring.servlet.multipart.max-request-size=25MB \ No newline at end of file diff --git a/src/main/resources/static/aufnahme.html b/src/main/resources/static/aufnahme.html index f86ec5c..aff00f3 100644 --- a/src/main/resources/static/aufnahme.html +++ b/src/main/resources/static/aufnahme.html @@ -64,13 +64,14 @@ const audioBlob = new Blob(chunks, { type: "audio/webm" }); const formData = new FormData(); - formData.append("file", audioBlob, "aufnahme.webm"); + formData.append("audio", audioBlob, "aufnahme.webm"); try { - const response = await fetch("/interview/transcription", { method: "POST", body: formData }); + const response = await fetch("/knowledge-base/transcription", { method: "POST", body: formData }); if (!response.ok) throw new Error("Server: " + response.status); - transcript.value = await response.text(); + const data = await response.json(); + transcript.value = data.text; status.textContent = "✅ Fertig – Text bitte prüfen und ggf. korrigieren."; saveBtn.style.display = "inline-block"; } catch (err) { @@ -91,14 +92,14 @@ status.textContent = "⏳ Speichern läuft…"; try { - const response = await fetch("/interview/save", { + const fd = new FormData(); + fd.append("text", transcript.value); + fd.append("title", document.getElementById("interviewer").value); + fd.append("city", document.getElementById("city").value); + + const response = await fetch("/knowledge-base/save-new-interview", { method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - interviewr: document.getElementById("interviewer").value, - interviewTranscript: transcript.value, - city: document.getElementById("city").value - }) + body: fd }); const antwort = await response.json(); // ApiResponse: {success, errorMessage, data}