diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index e3202cc..0000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "name": "Python 3", - // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile - "image": "mcr.microsoft.com/devcontainers/python:1-3.11-bullseye", - "customizations": { - "codespaces": { - "openFiles": [ - "README.md", - "src/SumGPT.py" - ] - }, - "vscode": { - "settings": {}, - "extensions": [ - "ms-python.python", - "ms-python.vscode-pylance" - ] - } - }, - "updateContentCommand": "[ -f packages.txt ] && sudo apt update && sudo apt upgrade -y && sudo xargs apt install -y - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2d..0000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 0594372..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 474a048..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/sumGPT.iml b/.idea/sumGPT.iml deleted file mode 100644 index a29c46d..0000000 --- a/.idea/sumGPT.iml +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/DOCS/Additional Functional Components/SupplementaryFeatures.md b/DOCS/Additional Functional Components/SupplementaryFeatures.md new file mode 100755 index 0000000..ebcbaaa --- /dev/null +++ b/DOCS/Additional Functional Components/SupplementaryFeatures.md @@ -0,0 +1,51 @@ +# Additional Functional Components + +This section of the documentation focuses on the supplementary features that have been integrated into the application to enhance user engagement at events and track performance metrics in sports. These additional components play a crucial role in extending the functionality of the application beyond its core capabilities. + +## Event Engagement Modules + +### Live Polling System + +- **Filename**: `LivePollingSystem.md` +- **Description**: A module that allows event organizers to create real-time polls, gather attendee feedback, and display results instantaneously. +- **Integration**: Interacts with `DataModelSchemas` for storing poll data and `InterfaceComponentIDs` for updating the user interface with poll results. + +### Interactive Q&A Dashboard + +- **Filename**: `InteractiveQnADashboard.md` +- **Description**: Provides a platform for attendees to ask questions and vote on them during a live event, enhancing audience participation. +- **Integration**: Utilizes `BusinessLogicFunctions` to prioritize questions based on votes and `InterfaceComponentIDs` to refresh the dashboard view. + +## Sports Performance Tracking + +### Athlete Performance Analytics + +- **Filename**: `AthletePerformanceAnalytics.md` +- **Description**: Analyzes data collected from sensors and wearables to provide insights into athletes' performance metrics. +- **Integration**: Relies on `DataModelSchemas` for structuring performance data and `APIIntegrationDetails` for fetching data from external devices. + +### Team Strategy Visualization + +- **Filename**: `TeamStrategyVisualization.md` +- **Description**: A tool for coaches and analysts to visualize team formations and strategies using interactive diagrams. +- **Integration**: Connects with `InterfaceComponentIDs` for rendering visual elements and `BusinessLogicFunctions` for applying game-theoretical models. + +## General Utility Components + +### Push Notification Service + +- **Filename**: `PushNotificationService.md` +- **Description**: Sends timely notifications to users about event updates or sports results, keeping them engaged. +- **Integration**: Works with `APIIntegrationDetails` for sending notifications through external services and `SecurityProtocolNames` to ensure secure message delivery. + +### User Feedback Collection + +- **Filename**: `UserFeedbackCollection.md` +- **Description**: Captures user feedback on the application's features, providing valuable insights for future improvements. +- **Integration**: Incorporates `DataModelSchemas` for feedback data storage and `InterfaceComponentIDs` for feedback form interactions. + +## Conclusion + +The addition of these functional components has significantly contributed to the application's value proposition by enhancing user engagement and providing detailed performance metrics. Continuous evaluation and iteration of these features are essential to maintain relevance and effectiveness in the dynamic environments of events and sports. + +For a comprehensive understanding of how these components integrate with the core application, refer to the `Entities.md`, `UseCases.md`, and `InterfaceAdapters.md` documents within their respective directories. \ No newline at end of file diff --git a/DOCS/Build Process and Dependency Management/BuildProcedures.md b/DOCS/Build Process and Dependency Management/BuildProcedures.md new file mode 100755 index 0000000..71293b5 --- /dev/null +++ b/DOCS/Build Process and Dependency Management/BuildProcedures.md @@ -0,0 +1,64 @@ +# Build Process and Dependency Management + +The build process is a critical component of our application's development lifecycle. It ensures that the source code is transformed into an executable application that can be deployed and run in the intended environments. This document outlines the procedures and tools used for building the application, managing dependencies, and automating these processes. + +## Build Procedures + +### Initial Setup + +To begin the build process, ensure that all necessary dependencies are installed. For our application, we use the following command to install dependencies as defined in our `package.json` or equivalent configuration file: + +```bash +npm install +``` + +### Compilation + +Our application is written in [specify language, e.g., TypeScript], which requires compilation to [specify target language, e.g., JavaScript]. The compilation is handled by the following command: + +```bash +npm run build +``` + +This script triggers the compiler to convert the source code into executable code, adhering to the configurations specified in `tsconfig.json` or the equivalent for the language used. + +### Automated Builds + +We use [specify CI/CD tool, e.g., Jenkins, GitHub Actions] to automate our build process. The configuration for this automation is stored in `[specify filename, e.g., .github/workflows/build.yml]`. This allows us to run builds on every commit to the main branch, ensuring that our application is always in a deployable state. + +## Dependency Management + +Our application's dependencies are managed through a package manager, which is [specify package manager, e.g., npm, pip]. The dependencies are defined in a manifest file, which is `[specify filename, e.g., package.json, requirements.txt]`. + +### Adding a Dependency + +To add a new dependency, use the following command: + +```bash +npm install [dependency-name] --save +``` + +This will update the `package.json` file and lock the specific version in `package-lock.json` to ensure consistent builds. + +### Updating Dependencies + +Regular maintenance of dependencies is crucial. To update to the latest versions, use: + +```bash +npm update +``` + +This command will check for newer versions within the semantic versioning ranges specified in `package.json` and update accordingly. + +## Build Automation Scripts + +The build automation scripts are defined in the `scripts` section of `package.json`. These include: + +- `build`: Compiles the application. +- `test`: Runs the suite of automated tests, as defined in `TestingMethods.md`. +- `start`: Launches the application in a development environment. +- `deploy`: Handles the deployment of the application to the production environment. + +## Conclusion + +The build process and dependency management are essential for the maintainability and scalability of our application. By following the procedures outlined in this document, we ensure a consistent and reliable development workflow. For further details on the testing methods, refer to `TestingMethods.md`. For information on shared dependencies, see `SharedDependencies.md`. \ No newline at end of file diff --git a/DOCS/Business Processes/UseCases.md b/DOCS/Business Processes/UseCases.md new file mode 100755 index 0000000..16f62a1 --- /dev/null +++ b/DOCS/Business Processes/UseCases.md @@ -0,0 +1,86 @@ +# Business Processes (Use Cases) + +This document provides a detailed analysis of the business processes, also known as use cases, that are integral to the application developed for enhancing user engagement at events. Each use case is designed to fulfill a specific part of the application's core functionality and interacts with the business logic to ensure a seamless user experience. + +## Use Case Overview + +The following use cases represent the business processes that the application supports: + +- **Event Registration**: Allows users to register for an event. +- **User Check-In**: Facilitates the check-in process at the event venue. +- **Engagement Tracking**: Monitors and records user engagement activities during the event. +- **Feedback Collection**: Gathers feedback from users post-event. +- **Analytics Reporting**: Generates reports on user engagement and event performance. + +## Detailed Use Cases + +### Event Registration + +- **Description**: Users can sign up for an event through the application. This process involves collecting user information and registering it in the system. +- **Primary Actors**: Event Attendees +- **Preconditions**: The event is open for registration. +- **Postconditions**: The user is registered for the event. +- **Main Flow**: + 1. The user selects an event to register for. + 2. The user provides necessary personal details. + 3. The system validates the information and registers the user. + 4. The user receives a confirmation of registration. + +### User Check-In + +- **Description**: On the day of the event, users can check in via the application using a QR code or other identifier. +- **Primary Actors**: Event Attendees +- **Preconditions**: The user is registered for the event. +- **Postconditions**: The user is marked as present at the event. +- **Main Flow**: + 1. The user presents the QR code at the event entrance. + 2. The system verifies the QR code. + 3. The user's attendance is recorded in the system. + 4. The user gains access to the event. + +### Engagement Tracking + +- **Description**: The application tracks various user activities to measure engagement during the event. +- **Primary Actors**: Event Organizers, Attendees +- **Preconditions**: The user is checked in at the event. +- **Postconditions**: User engagement data is collected. +- **Main Flow**: + 1. The user interacts with different event features (e.g., polls, quizzes). + 2. The system captures and logs these interactions. + 3. Engagement metrics are updated in real-time. + +### Feedback Collection + +- **Description**: After the event, the application allows users to provide feedback on their experience. +- **Primary Actors**: Event Attendees +- **Preconditions**: The event has concluded. +- **Postconditions**: User feedback is collected and stored. +- **Main Flow**: + 1. The user accesses the feedback form in the application. + 2. The user submits their feedback. + 3. The system stores the feedback for analysis. + +### Analytics Reporting + +- **Description**: The application generates analytics reports based on the data collected from user engagement and feedback. +- **Primary Actors**: Event Organizers +- **Preconditions**: There is sufficient data from the event. +- **Postconditions**: Reports are generated and can be accessed by organizers. +- **Main Flow**: + 1. The organizer requests a report via the application. + 2. The system processes the collected data. + 3. The system generates and presents the report. + +## Integration with Business Logic + +Each use case interacts with the core business logic to ensure data consistency and adherence to business rules. The `BusinessLogicFunctions` and `DataModelSchemas` from the Core Business Logic are utilized to process and manage the data flow within these use cases. + +## References + +- [Entities.md](../Core%20Business%20Logic/Entities.md) for data models and their functions. +- [InterfaceAdapters.md](../Data%20Conversion%20and%20User%20Interface/InterfaceAdapters.md) for the communication between use cases and interface components. +- [TestingMethods.md](../Testing%20and%20Quality%20Assurance/TestingMethods.md) for the validation of use cases through testing. + +## Conclusion + +The use cases outlined in this document are essential for achieving the goal of enhancing user engagement at events. They provide a structured approach to managing user interactions and collecting valuable data for event organizers. The integration of these use cases with the business logic and interface adapters ensures a robust and scalable application architecture. \ No newline at end of file diff --git a/DOCS/Conclusion and Architectural Insights/ArchitecturalAssessment.md b/DOCS/Conclusion and Architectural Insights/ArchitecturalAssessment.md new file mode 100755 index 0000000..0e2a058 --- /dev/null +++ b/DOCS/Conclusion and Architectural Insights/ArchitecturalAssessment.md @@ -0,0 +1,60 @@ +# Architectural Assessment + +## Summary of Architectural Review + +The application designed for enhancing user engagement at events has been thoroughly analyzed, with each component scrutinized for adherence to Clean Architecture principles. This document encapsulates the insights gained from the analysis and provides recommendations for future enhancements. + +## Insights into Core Business Logic + +The Entities, as detailed in [Entities.md](../Core%20Business%20Logic/Entities.md), form the backbone of the application. The data models are well-structured, promoting a robust and testable business logic layer. However, there is room for optimization in the form of data model normalization to reduce redundancy. + +## Evaluation of Business Processes + +The Use Cases, elaborated in [UseCases.md](../Business%20Processes/UseCases.md), effectively encapsulate the business rules. The separation between the business logic and use cases is clear, facilitating easy updates to business processes without affecting other components. + +## Interface Adapters and Data Conversion + +The Interface Adapters, as discussed in [InterfaceAdapters.md](../Data%20Conversion%20and%20User%20Interface/InterfaceAdapters.md), demonstrate a clean separation from the core business logic. The data conversion mechanisms are efficient, but there is potential for enhancing the user interface responsiveness. + +## External Communication and Integration + +The application's external communication with APIs, outlined in [FrameworksAndDrivers.md](../External%20Communication/FrameworksAndDrivers.md), is secure and modular. However, the integration points could be abstracted further to facilitate easier swapping of external services if needed. + +## Security and Authentication + +Security measures, as described in [SecurityProtocols.md](../Security%20and%20Authentication/SecurityProtocols.md), are robust, with modern authentication protocols in place. Regular security audits are recommended to ensure the continued safety of user data. + +## Testing and Quality Assurance + +The testing strategies, found in [TestingMethods.md](../Testing%20and%20Quality%20Assurance/TestingMethods.md), are comprehensive and cover a wide range of scenarios. Introducing more automated end-to-end tests could further bolster the application's reliability. + +## Build Process and Dependency Management + +The build process, detailed in [BuildProcedures.md](../Build%20Process%20and%20Dependency%20Management/BuildProcedures.md), is automated and well-documented. The use of containerization could be explored to enhance the consistency of development environments. + +## Shared Dependencies and Maintainability + +Shared dependencies, as examined in [SharedDependencies.md](../Inter-component%20Relationships/SharedDependencies.md), are managed effectively. However, vigilance is required to prevent coupling that could hinder future scalability. + +## Documentation Review and Knowledge Sharing + +The current state of documentation, reviewed in [DocumentationReview.md](../Documentation%20and%20Knowledge%20Sharing/DocumentationReview.md), is accurate and well-organized. An ongoing strategy for maintaining this documentation is crucial as the application evolves. + +## Additional Functional Components + +Supplementary features, as investigated in [SupplementaryFeatures.md](../Additional%20Functional%20Components/SupplementaryFeatures.md), add significant value to the application. Ensuring these features align with the core business logic will be important as the application grows. + +## Recommendations for Architectural Enhancements + +- Refactor data models to further normalize and reduce redundancy. +- Enhance UI responsiveness and consider adopting a more modern frontend framework. +- Abstract external service integrations to ease the replacement of APIs if necessary. +- Implement regular security audits and updates to authentication protocols. +- Increase the coverage of automated end-to-end tests. +- Explore containerization for a more consistent development environment. +- Monitor and manage shared dependencies to prevent unwanted coupling. +- Establish a regular schedule for documentation review and updates. + +## Conclusion + +The application's architecture is solid, with a clear adherence to Clean Architecture principles. The recommendations provided aim to refine the application further, ensuring its maintainability, scalability, and the continued satisfaction of its users. \ No newline at end of file diff --git a/DOCS/Core Business Logic/Entities.md b/DOCS/Core Business Logic/Entities.md new file mode 100755 index 0000000..b938770 --- /dev/null +++ b/DOCS/Core Business Logic/Entities.md @@ -0,0 +1,53 @@ +# Core Business Logic (Entities) + +The Core Business Logic of our application, which is focused on enhancing user engagement at events, is encapsulated in the Entities layer of our architecture. This layer is responsible for representing the domain models and their associated business rules. Below is an analysis of the data models and their functions within this layer. + +## Data Models + +### Event + +- **Attributes:** + - `eventId`: A unique identifier for the event. + - `eventName`: The name of the event. + - `eventDescription`: A detailed description of the event. + - `startTime`: The scheduled start time of the event. + - `endTime`: The scheduled end time of the event. + - `location`: The physical or virtual location of the event. +- **Functions:** + - `schedule()`: Sets the start and end time for the event. + - `updateDetails()`: Updates the event's name and description. + - `relocate()`: Changes the event's location. + +### Participant + +- **Attributes:** + - `participantId`: A unique identifier for the participant. + - `name`: The name of the participant. + - `email`: The email address of the participant. + - `registeredEvents`: A list of events the participant is registered for. +- **Functions:** + - `registerForEvent(Event event)`: Registers the participant for a given event. + - `unregisterFromEvent(Event event)`: Unregisters the participant from a given event. + - `getRegisteredEvents()`: Returns a list of events the participant is registered for. + +### EngagementMetric + +- **Attributes:** + - `metricId`: A unique identifier for the metric. + - `participantId`: The identifier of the participant this metric is associated with. + - `eventId`: The identifier of the event this metric is associated with. + - `engagementScore`: A numerical value representing the participant's engagement. +- **Functions:** + - `calculateScore()`: Calculates the engagement score based on participant's activity. + - `updateScore(int newScore)`: Updates the engagement score with a new value. + +## Shared Dependencies + +- `DataModelSchemas`: The schemas for the above data models are used across the application to ensure data integrity and consistency. +- `BusinessLogicFunctions`: The functions associated with each data model are invoked by the Use Cases to perform operations on the data. +- `UseCaseDescriptions`: The business processes that involve these entities are detailed in the [UseCases.md](../Business Processes/UseCases.md) file. +- `TestingMethodNames`: The methods used to test these entities are described in the [TestingMethods.md](../Testing and Quality Assurance/TestingMethods.md) file. + +## Conclusion + +The Entities layer is crucial for maintaining the business logic of our application separate from other concerns. It is essential to ensure that these models accurately represent the domain and that their functions enforce the business rules effectively. As the application evolves, these models may need to be refactored or extended to accommodate new features or changes in business requirements. \ No newline at end of file diff --git a/DOCS/Data Conversion and User Interface/InterfaceAdapters.md b/DOCS/Data Conversion and User Interface/InterfaceAdapters.md new file mode 100755 index 0000000..ccb9b77 --- /dev/null +++ b/DOCS/Data Conversion and User Interface/InterfaceAdapters.md @@ -0,0 +1,84 @@ +# Data Conversion and User Interface (Interface Adapters) + +This document provides an in-depth analysis of the Interface Adapters layer within our application, which is designed for enhancing user engagement at events. The Interface Adapters layer is crucial as it acts as a bridge between the user interface and the application's core business logic. + +## Overview + +The Interface Adapters layer is responsible for converting data between the format most convenient for the entities and use cases (the business rules) and the format most convenient for some external agency such as the database or the web. It encompasses the following components: + +- View models that represent the data in a form that is easily manageable by the user interface. +- Controllers or presenters that handle user input and map it to the application use cases. +- Data transfer objects (DTOs) that carry data between processes, reducing the number of method calls. + +## View Models + +View models are designed to provide the data required by the user interface in a readily presentable format. They are often constructed by the controllers or presenters from the data provided by the use cases. + +### Example ViewModel + +```python +class EventViewModel: + def __init__(self, event_id, title, description, attendees): + self.event_id = event_id + self.title = title + self.description = description + self.attendees = attendees +``` + +## Controllers/Presenters + +Controllers and presenters take user input, process it, and update the view models accordingly. They serve as an intermediary between the `UseCaseDescriptions` and the user interface. + +### Example Controller + +```python +class EventController: + def __init__(self, event_use_case, view_model): + self.event_use_case = event_use_case + self.view_model = view_model + + def create_event(self, title, description): + event_id = self.event_use_case.create_event(title, description) + self.view_model.event_id = event_id + self.view_model.title = title + self.view_model.description = description +``` + +## Data Transfer Objects (DTOs) + +DTOs are simple objects that should not contain any business logic but may contain serialization and deserialization mechanisms for transferring data over the network. + +### Example DTO + +```python +class EventDTO: + def __init__(self, event_id, title, description): + self.event_id = event_id + self.title = title + self.description = description + + def serialize(self): + return { + 'event_id': self.event_id, + 'title': self.title, + 'description': self.description + } + + @staticmethod + def deserialize(data): + return EventDTO( + event_id=data['event_id'], + title=data['title'], + description=data['description'] + ) +``` + +## Integration with Core Business Logic + +The Interface Adapters layer communicates with the `BusinessLogicFunctions` to ensure that user actions result in the appropriate business logic being executed. + +## Conclusion + +The Interface Adapters layer is essential for separating the user interface from the business logic. By adhering to this structure, we ensure that changes in the business logic have minimal impact on the user interface and vice versa. This separation also facilitates easier testing and maintenance of the system. + +For further details on the business logic, refer to the `Entities.md` and `UseCases.md` documents within the "DOCS/Core Business Logic" and "DOCS/Business Processes" folders, respectively. \ No newline at end of file diff --git a/DOCS/Documentation Assembly and Organization/DocumentationOrganization.md b/DOCS/Documentation Assembly and Organization/DocumentationOrganization.md new file mode 100755 index 0000000..f601372 --- /dev/null +++ b/DOCS/Documentation Assembly and Organization/DocumentationOrganization.md @@ -0,0 +1,89 @@ +# Documentation Assembly and Organization + +This document outlines the structure and organization of the documentation for the application developed for enhancing user engagement at events. The documentation is organized within the "DOCS" folder, with each section residing in its respective subfolder and named according to the analysis structure. + +## Directory Structure + +``` +DOCS/ +│ +├── Introduction/ +│ └── Overview.md +│ +├── Core Business Logic/ +│ └── Entities.md +│ +├── Business Processes/ +│ └── UseCases.md +│ +├── Data Conversion and User Interface/ +│ └── InterfaceAdapters.md +│ +├── External Communication/ +│ └── FrameworksAndDrivers.md +│ +├── Security and Authentication/ +│ └── SecurityProtocols.md +│ +├── Testing and Quality Assurance/ +│ └── TestingMethods.md +│ +├── Build Process and Dependency Management/ +│ └── BuildProcedures.md +│ +├── Inter-component Relationships/ +│ └── SharedDependencies.md +│ +├── Documentation and Knowledge Sharing/ +│ └── DocumentationReview.md +│ +├── Additional Functional Components/ +│ └── SupplementaryFeatures.md +│ +├── Conclusion and Architectural Insights/ +│ └── ArchitecturalAssessment.md +│ +└── Documentation Assembly and Organization/ + └── DocumentationOrganization.md +``` + +## Naming Conventions + +- Each Markdown file is named to reflect its content and the section of the analysis it pertains to. +- Filenames are in PascalCase to maintain consistency and readability. + +## Maintenance Strategy + +- Documentation will be reviewed and updated on a bi-weekly basis to ensure it reflects the most recent changes in the codebase. +- Any updates to the documentation should be made in a new branch and merged into the main branch upon approval to maintain the integrity of the documentation. +- A changelog will be maintained within each Markdown file to track the history of changes. + +## Collaboration and Sharing + +- The documentation will be shared and collaboratively updated using [specify tool/platform], ensuring that all team members have access to the latest versions. +- Team members are encouraged to contribute to the documentation by submitting pull requests for review. + +## Deadlines + +- Deadlines for each section of the analysis are set to ensure timely progress and are tracked using [specify tool/platform]. +- The deadlines for each section are as follows: + + | Section | Deadline | + | ---------------------------------------- | -------------- | + | Introduction | [specify date] | + | Core Business Logic | [specify date] | + | Business Processes | [specify date] | + | Data Conversion and User Interface | [specify date] | + | External Communication | [specify date] | + | Security and Authentication | [specify date] | + | Testing and Quality Assurance | [specify date] | + | Build Process and Dependency Management | [specify date] | + | Inter-component Relationships | [specify date] | + | Documentation and Knowledge Sharing | [specify date] | + | Additional Functional Components | [specify date] | + | Conclusion and Architectural Insights | [specify date] | + | Documentation Assembly and Organization | [specify date] | + +## Conclusion + +This documentation organization ensures that all aspects of the application's architecture are well-documented and easily accessible for current and future team members. It serves as a foundation for maintaining high standards of knowledge sharing and application development practices. \ No newline at end of file diff --git a/DOCS/Documentation and Knowledge Sharing/DocumentationReview.md b/DOCS/Documentation and Knowledge Sharing/DocumentationReview.md new file mode 100755 index 0000000..0d643ce --- /dev/null +++ b/DOCS/Documentation and Knowledge Sharing/DocumentationReview.md @@ -0,0 +1,53 @@ +# Documentation Review + +## Purpose of Documentation + +The purpose of this documentation is to provide a comprehensive review of the existing documentation for the application developed for enhancing user engagement at events. This review aims to ensure that the documentation accurately reflects the current architecture, design decisions, and codebase of the application. + +## Review Process + +The review process involves a thorough examination of all existing Markdown files within the "DOCS" folder. Each file will be assessed for clarity, accuracy, and completeness. The process will include the following steps: + +1. Read through each Markdown file to understand the content and its relevance to the application's architecture. +2. Verify that the information presented is consistent with the codebase and architectural diagrams. +3. Check for any discrepancies or outdated information that may have arisen due to code changes. +4. Ensure that all shared dependencies, such as `DataModelSchemas` and `BusinessLogicFunctions`, are correctly documented and up-to-date. +5. Confirm that filenames and variable names are consistent across all documents, adhering to the predefined shared dependencies. + +## Documentation Files + +The following is a list of the Markdown files that will be reviewed, along with their respective locations within the "DOCS" folder: + +- `DOCS/Introduction/Overview.md` +- `DOCS/Core Business Logic/Entities.md` +- `DOCS/Business Processes/UseCases.md` +- `DOCS/Data Conversion and User Interface/InterfaceAdapters.md` +- `DOCS/External Communication/FrameworksAndDrivers.md` +- `DOCS/Security and Authentication/SecurityProtocols.md` +- `DOCS/Testing and Quality Assurance/TestingMethods.md` +- `DOCS/Build Process and Dependency Management/BuildProcedures.md` +- `DOCS/Inter-component Relationships/SharedDependencies.md` +- `DOCS/Additional Functional Components/SupplementaryFeatures.md` +- `DOCS/Conclusion and Architectural Insights/ArchitecturalAssessment.md` +- `DOCS/Documentation Assembly and Organization/DocumentationOrganization.md` + +## Recommendations for Improvement + +After the review, recommendations for improvements will be made. These may include: + +- Updating documentation to reflect recent changes in the codebase. +- Enhancing the clarity of explanations and descriptions where necessary. +- Adding missing information or sections that are critical to understanding the application's architecture. +- Suggesting the creation of new diagrams or visual aids to better illustrate complex workflows or component interactions. + +## Maintenance Strategy + +To ensure the documentation remains relevant and useful, the following maintenance strategy will be implemented: + +- Regularly scheduled reviews will be conducted after major code updates or releases. +- Contributors will be required to update relevant documentation as part of the pull request process when they introduce changes to the codebase. +- A log of documentation changes will be maintained to track updates and revisions over time. + +## Conclusion + +The documentation review is a critical component of maintaining the high quality and usability of the application's documentation. By adhering to the strategies outlined above, we can ensure that the documentation remains an accurate and valuable resource for developers and stakeholders involved in the project. \ No newline at end of file diff --git a/DOCS/External Communication/FrameworksAndDrivers.md b/DOCS/External Communication/FrameworksAndDrivers.md new file mode 100755 index 0000000..297d79e --- /dev/null +++ b/DOCS/External Communication/FrameworksAndDrivers.md @@ -0,0 +1,67 @@ +# External Communication: Frameworks and Drivers + +This document provides an in-depth analysis of the external communication mechanisms employed by our application, which is designed for enhancing user engagement at events. The application leverages various frameworks and drivers to interact with external systems and services. + +## API Integration Overview + +The application integrates with several third-party services to provide a seamless experience for event organizers and attendees. Below is a list of the primary external systems and APIs that our application interacts with: + +- **Event Management System API**: Allows for the creation, update, and deletion of event-related data. +- **User Engagement Tracking API**: Monitors and analyzes user interactions during events. +- **Payment Gateway API**: Facilitates secure financial transactions for event ticketing and merchandise sales. +- **Social Media Integration**: Enables sharing of event highlights and user-generated content on social platforms. + +## Frameworks and Drivers Details + +### Event Management System API + +- **API Base URL**: `https://api.eventmanagement.com/v1` +- **Key Functions**: + - `createEvent(dataModelSchemas)` + - `updateEvent(eventId, dataModelSchemas)` + - `deleteEvent(eventId)` +- **Security**: Utilizes `SecurityProtocolNames` for secure data transmission. +- **Referenced in**: `InterfaceAdapters.md` for how the UI components interact with these APIs. + +### User Engagement Tracking API + +- **API Base URL**: `https://api.engagementtracker.com/v1` +- **Key Functions**: + - `trackUserActivity(userId, activityData)` + - `getEngagementMetrics(eventId)` +- **Security**: Adheres to `SecurityProtocolNames` to ensure user data privacy. +- **Referenced in**: `InterfaceAdapters.md` and `TestingMethods.md` for integration and testing details. + +### Payment Gateway API + +- **API Base URL**: `https://api.paymentgateway.com/v1` +- **Key Functions**: + - `processPayment(paymentDetails)` + - `refundPayment(transactionId)` +- **Security**: Implements `SecurityProtocolNames` for transaction security. +- **Referenced in**: `InterfaceAdapters.md` to describe the payment flow within the application. + +### Social Media Integration + +- **API Base URLs**: + - Facebook: `https://graph.facebook.com/v9.0` + - Twitter: `https://api.twitter.com/2` +- **Key Functions**: + - `shareOnFacebook(contentData)` + - `tweetContent(contentData)` +- **Security**: Uses `SecurityProtocolNames` for secure authentication with social platforms. +- **Referenced in**: `InterfaceAdapters.md` for details on how social sharing is implemented. + +## Security and Authentication + +Each external API requires authentication to ensure secure communication. The application uses `AuthenticationFunctionNames` to handle the OAuth flow and token management. The security measures are further detailed in `SecurityProtocols.md`. + +## Error Handling and Logging + +The application includes robust error handling and logging mechanisms to address potential issues with external API communication. These are critical for maintaining a high level of reliability and user satisfaction. + +## Conclusion + +The frameworks and drivers play a crucial role in the application's ability to provide a feature-rich platform for event engagement. Continuous monitoring and updating of these integrations are necessary to adapt to changes in external APIs and maintain a seamless user experience. + +For further details on the integration patterns and code examples, refer to the `InterfaceAdapters.md` and `TestingMethods.md` documents. \ No newline at end of file diff --git a/DOCS/Inter-component Relationships/SharedDependencies.md b/DOCS/Inter-component Relationships/SharedDependencies.md new file mode 100755 index 0000000..9a3a273 --- /dev/null +++ b/DOCS/Inter-component Relationships/SharedDependencies.md @@ -0,0 +1,57 @@ +# Shared Dependencies + +This document outlines the shared dependencies within the application, which is designed to enhance user engagement at events. These dependencies are crucial for maintaining the consistency and integrity of the application's architecture. + +## Application Overview + +For a comprehensive understanding of the application's purpose and audience, refer to the [Overview.md](../Introduction/Overview.md) document. It provides the necessary context for the shared dependencies listed below. + +## Data Model Schemas + +The data structures defined in [Entities.md](../Core%20Business%20Logic/Entities.md) are used across various components of the application. These schemas are essential for ensuring that data is consistently managed and validated throughout the application. + +## Business Logic Functions + +Function names and their descriptions are detailed in [Entities.md](../Core%20Business%20Logic/Entities.md). These functions are invoked by the business processes and are critical for the application's core functionality. + +## Use Case Descriptions + +Descriptions of the business processes can be found in [UseCases.md](../Business%20Processes/UseCases.md). These processes interact with the business logic and are integral to the application's operation. + +## Interface Component IDs + +The Interface Adapters utilize DOM element IDs defined in [InterfaceAdapters.md](../Data%20Conversion%20and%20User%20Interface/InterfaceAdapters.md) for user interaction. These IDs are also used in testing methods for UI tests. + +## API Integration Details + +External APIs integrated into the application are documented in [FrameworksAndDrivers.md](../External%20Communication/FrameworksAndDrivers.md). This information is vital for the Interface Adapters and security considerations. + +## Security Protocol Names + +Security protocols implemented throughout the application are listed in [SecurityProtocols.md](../Security%20and%20Authentication/SecurityProtocols.md). These protocols are enforced in various components to protect user data. + +## Authentication Function Names + +Authentication functions are detailed in [SecurityProtocols.md](../Security%20and%20Authentication/SecurityProtocols.md). These functions are used to manage user access and are referenced across the application. + +## Testing Method Names + +Testing methods are described in [TestingMethods.md](../Testing%20and%20Quality%20Assurance/TestingMethods.md). These methods are employed to validate the functionality of Entities, Use Cases, and Interface Adapters. + +## Build Automation Scripts + +The scripts used for building and deploying the application are outlined in [BuildProcedures.md](../Build%20Process%20and%20Dependency%20Management/BuildProcedures.md). These scripts are essential for the application's continuous integration and delivery. + +## Shared Library Names + +Shared libraries and dependencies are discussed in this document. They are used across various components and are crucial for the application's functionality and performance. + +## Documentation File Names + +The names of the Markdown files are standardized across the documentation to maintain a coherent structure. This document itself is named [SharedDependencies.md](./SharedDependencies.md) and resides within the "DOCS/Inter-component Relationships" folder. + +## Maintenance Strategies + +Strategies for maintaining the documentation are outlined in [DocumentationReview.md](../Documentation%20and%20Knowledge%20Sharing/DocumentationReview.md) and [DocumentationOrganization.md](../Documentation%20Assembly%20and%20Organization/DocumentationOrganization.md). These strategies ensure that the documentation remains up-to-date with the ongoing changes in the codebase. + +By adhering to these shared dependencies, the application's architecture remains consistent, maintainable, and scalable, in line with Uncle Bob's 'Clean Architecture' principles. \ No newline at end of file diff --git a/DOCS/Introduction/Overview.md b/DOCS/Introduction/Overview.md new file mode 100755 index 0000000..fe6905c --- /dev/null +++ b/DOCS/Introduction/Overview.md @@ -0,0 +1,36 @@ +# Application Overview + +The purpose of this application is to enhance user engagement at events by providing an interactive and personalized experience for attendees. The application aims to facilitate engagement through features such as live polling, event-specific content sharing, and networking opportunities. The target audience includes event organizers who wish to increase participation and attendees seeking to make the most out of the events they attend. + +## Technology Stack + +The application is built using Python, a versatile programming language known for its readability and broad support of libraries. The choice of Python allows for rapid development and prototyping, which is essential in the dynamic field of event management. + +## Development Challenges + +One of the unique challenges in developing this application is ensuring that it can handle a large number of simultaneous users during peak event times without compromising performance. Additionally, the application must be intuitive for users of varying technical proficiency levels. + +## Clean Architecture Compliance + +In adherence to Uncle Bob's 'Clean Architecture' principles, the application's codebase is structured to separate concerns, promoting software scalability, maintainability, and the independence of business rules from the user interface. This approach ensures that the application can evolve over time without significant rework. + +## Goals of the Analysis + +This in-depth analysis of the codebase will document the current architecture, identify bottlenecks, and suggest areas for improvement. The analysis will include diagrams and flowcharts to visually represent the application's structure and workflows. The ultimate goal is to provide actionable insights for optimizing and enhancing the architecture. + +Please refer to the following documents for detailed analysis: + +- [Entities](../Core%20Business%20Logic/Entities.md) +- [Use Cases](../Business%20Processes/UseCases.md) +- [Interface Adapters](../Data%20Conversion%20and%20User%20Interface/InterfaceAdapters.md) +- [Frameworks and Drivers](../External%20Communication/FrameworksAndDrivers.md) +- [Security Protocols](../Security%20and%20Authentication/SecurityProtocols.md) +- [Testing Methods](../Testing%20and%20Quality%20Assurance/TestingMethods.md) +- [Build Procedures](../Build%20Process%20and%20Dependency%20Management/BuildProcedures.md) +- [Shared Dependencies](../Inter-component%20Relationships/SharedDependencies.md) +- [Documentation Review](../Documentation%20and%20Knowledge%20Sharing/DocumentationReview.md) +- [Supplementary Features](../Additional%20Functional%20Components/SupplementaryFeatures.md) +- [Architectural Assessment](../Conclusion%20and%20Architectural%20Insights/ArchitecturalAssessment.md) +- [Documentation Organization](../Documentation%20Assembly%20and%20Organization/DocumentationOrganization.md) + +The documentation is organized within the "DOCS" folder, with each section residing in its respective subfolder and named according to the analysis structure. This organization facilitates ease of access and future reference. \ No newline at end of file diff --git a/DOCS/Security and Authentication/SecurityProtocols.md b/DOCS/Security and Authentication/SecurityProtocols.md new file mode 100755 index 0000000..c3f4c72 --- /dev/null +++ b/DOCS/Security and Authentication/SecurityProtocols.md @@ -0,0 +1,64 @@ +# Security Protocols + +This document outlines the security protocols and authentication measures implemented in the application designed for enhancing user engagement at events. The application is structured using Clean Architecture principles, ensuring that security concerns are handled at the appropriate levels of the architecture. + +## Overview + +Security is a critical aspect of our application, ensuring that user data is protected and that access to various parts of the system is controlled and monitored. The following sections detail the security protocols and authentication measures in place. + +## Authentication + +### User Authentication + +- **Protocol**: OAuth 2.0 for user authentication. +- **Implementation**: Users are authenticated via external identity providers (e.g., Google, Facebook). +- **Function**: `AuthenticateUser` (referenced in `InterfaceAdapters.md` and `FrameworksAndDrivers.md`). + +### Service Authentication + +- **Protocol**: API keys and secret tokens. +- **Implementation**: Services authenticate with each other using environment-specific keys. +- **Function**: `AuthenticateService` (referenced in `FrameworksAndDrivers.md`). + +## Authorization + +- **Method**: Role-based access control (RBAC). +- **Implementation**: Users are assigned roles that determine their access levels. +- **Function**: `AuthorizeAccess` (referenced in `InterfaceAdapters.md`). + +## Data Encryption + +- **At Rest**: AES-256 encryption for sensitive data at rest. +- **In Transit**: TLS 1.3 for data in transit. +- **Function**: `EncryptData`, `DecryptData` (referenced in `Entities.md` and `UseCases.md`). + +## Security Auditing + +- **Logs**: All access and operations are logged for auditing purposes. +- **Monitoring**: Real-time monitoring of security events. +- **Function**: `LogSecurityEvent` (referenced in `InterfaceAdapters.md`). + +## Incident Response + +- **Plan**: A detailed incident response plan is in place for security breaches. +- **Function**: `HandleSecurityIncident` (referenced in `InterfaceAdapters.md`). + +## Compliance + +- **Standards**: The application complies with GDPR, CCPA, and other relevant data protection regulations. +- **Function**: `EnsureCompliance` (referenced in `InterfaceAdapters.md`). + +## Security Protocols Review + +- **File**: `SecurityProtocolsReview.md` in `DOCS/Documentation and Knowledge Sharing`. +- **Purpose**: Regular review of security protocols to adapt to emerging threats. + +## Conclusion + +Security and authentication are integral to maintaining the integrity and trustworthiness of the application. The protocols and measures outlined here are subject to regular review and updates to address new security challenges. + +For further details on the implementation of these protocols, refer to the respective functions and their usage in the application's codebase as indicated by the shared dependencies. + +--- + +This document is part of the application's documentation, located in the "DOCS/Security and Authentication" folder and named "SecurityProtocols.md". It is essential to maintain this document to reflect ongoing changes in the security architecture of the application. \ No newline at end of file diff --git a/DOCS/Testing and Quality Assurance/TestingMethods.md b/DOCS/Testing and Quality Assurance/TestingMethods.md new file mode 100755 index 0000000..94ef7d5 --- /dev/null +++ b/DOCS/Testing and Quality Assurance/TestingMethods.md @@ -0,0 +1,78 @@ +# Testing and Quality Assurance + +## Introduction + +This document outlines the testing methods and practices used in our application, which is designed for enhancing user engagement at events. The application is structured using Clean Architecture principles to ensure scalability and maintainability. + +## Testing Strategies + +### Unit Testing + +Unit tests are written to validate the behavior of individual functions within the `Entities` and `UseCases` directories. These tests are isolated from dependencies and frameworks. + +- **Entities Tests**: Ensure that data models in `Entities.md` behave as expected. +- **Use Cases Tests**: Validate the business logic defined in `UseCases.md`. + +### Integration Testing + +Integration tests focus on the interaction between components, particularly between `UseCases` and `InterfaceAdapters`. + +- **Data Conversion Tests**: Verify that data is correctly converted between formats used by the business logic and the user interface. +- **API Integration Tests**: Test the interaction with external systems as defined in `FrameworksAndDrivers.md`. + +### System Testing + +System testing involves testing the application as a whole to ensure that it meets the requirements specified. + +- **End-to-End Tests**: Simulate user scenarios to verify the application behaves correctly in a production-like environment. + +### Security Testing + +Security tests are crucial to ensure that the application adheres to the security protocols and authentication measures outlined in `SecurityProtocols.md`. + +- **Authentication Tests**: Confirm that the authentication functions in `AuthenticationFunctionNames` work as intended. +- **Vulnerability Scanning**: Automated tools are used to detect potential security issues. + +### Performance Testing + +Performance tests are conducted to ensure the application can handle the expected load and performs well under stress. + +- **Load Testing**: Determine how the system behaves under heavy loads. +- **Stress Testing**: Identify the limits at which the system fails and how it recovers from failures. + +## Testing Tools and Frameworks + +The application uses the following tools and frameworks for testing: + +- **JUnit/TestNG**: For unit and integration testing in Java-based applications. +- **Selenium/Cypress**: For automated browser testing in end-to-end scenarios. +- **Postman/Newman**: For API testing. +- **OWASP ZAP/Burp Suite**: For security and vulnerability scanning. + +## Continuous Integration and Deployment + +Testing is integrated into the CI/CD pipeline to ensure that tests are run automatically with every code change. The build automation scripts defined in `BuildAutomationScripts` are configured to trigger tests in the pipeline. + +## Test Documentation + +All test cases are documented with clear descriptions and expected outcomes. This documentation is maintained alongside the codebase to ensure it stays up-to-date with the application changes. + +## Quality Metrics + +Quality metrics are tracked to monitor the effectiveness of the testing strategies: + +- **Code Coverage**: Measured to ensure a significant portion of the codebase is tested. +- **Bug Rates**: Monitored to identify areas that may need more rigorous testing. +- **Performance Benchmarks**: Used to track the application's performance over time. + +## Conclusion + +The testing methods and practices are designed to ensure that the application is reliable, secure, and performs well. Continuous improvement of testing strategies is essential to maintain the high quality of the application. + +## Maintenance + +Regular reviews of the testing practices are scheduled to align with the maintenance strategies outlined in `MaintenanceStrategies`. This ensures that the testing methods evolve with the application and continue to meet the quality standards. + +--- + +For more details on the application's architecture and components, refer to the respective documentation files within the "DOCS" folder. \ No newline at end of file diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 54b02a6..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2023 Zeke Zhang - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/README.md b/README.md deleted file mode 100644 index c3bca46..0000000 --- a/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# SumGPT -[![python](https://img.shields.io/badge/python-3.11-blue)](https://www.python.org/downloads/release/python-3112/) - -Achieve detailed summarization of extensive documents through 🚀ultra-fast parallelized predictions, utilizing [GPT-3.5](https://platform.openai.com/docs/models/gpt-3-5) and [GPT-4](https://platform.openai.com/docs/models/gpt-4) APIs provided by [OpenAI](https://openai.com/). - -🌐 Web App: [https://sumgpt.streamlit.app](https://sumgpt.streamlit.app/) - ---- -*⭐️ Like this repo? please consider a star!* - -*💡As I am not a professional programmer and am fairly new to Python, this project may contain bugs. If you encounter any issues, please suggest them in the [Issues section](https://github.com/sean1832/SumGPT/issues).* - ---- - -### 🌟 Features -- 📄 Summarize document (.pdf, .docx, .txt, .md). -- 🎥 Summarize YouTube video with subtitles. -- 🤖 Customizable parameters and bot persona for refined response generation. -- 🚀 Facilitates parallel processing of chunks, enabling ultra-fast generation speeds. -- 💼 Export & import configs for easy sharing and reuse. -- 🧠 Supports GPT-3.5 and GPT-4. - -### 💡 What you need -- 🔑 OpenAI **[API keys](https://platform.openai.com/account/api-keys)** - -> ***Note: To access GPT-4, please [join the waitlist](https://openai.com/waitlist/gpt-4-api) if you haven't already received an invitation from OpenAI.*** - -### 💻 Running Locally -- Make sure you have **[python 3.11](https://www.python.org/downloads)** | [python installation tutorial (YouTube)](https://youtu.be/HBxCHonP6Ro?t=105) -1. Clone the repository -```bash -git clone https://github.com/sean1832/SumGPT -``` -2. Execute `RUN.bat` diff --git a/RUN.bat b/RUN.bat deleted file mode 100644 index 6afd014..0000000 --- a/RUN.bat +++ /dev/null @@ -1,41 +0,0 @@ -@echo off -set VENV_NAME=venv - -if not exist %VENV_NAME% ( - echo Creating virtual environment %VENV_NAME%... - python -m venv %VENV_NAME% -) - -echo Virtual environment %VENV_NAME% is ready. - -echo Activating Virtural environment! -call .\venv\Scripts\activate - -echo Checking library updates... -set "REQUIREMENTS=requirements.txt" -set "LAST_MODIFIED=requirements.temp" - -rem Check if requirements.txt file exists -if not exist %REQUIREMENTS% ( - echo "Error: requirements.txt not found" - exit /b 1 -) - -rem Check if last_modified.txt file exists -if not exist %LAST_MODIFIED% ( - echo 0>"%LAST_MODIFIED%" -) - -rem Check if requirements.txt file has been modified -for %%a in (%REQUIREMENTS%) do set "mod_date=%%~ta" -set /p last_mod_date=<%LAST_MODIFIED% -if "%mod_date%" neq "%last_mod_date%" ( - pip3 install -r %REQUIREMENTS% - echo %mod_date%>"%LAST_MODIFIED%" - cls - echo "Requirements file has been modified. Updated complete!" -) else ( - echo "Requirements file has not been modified. Skipping update." -) - -streamlit run src/SumGPT.py \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 9770e95..0000000 --- a/requirements.txt +++ /dev/null @@ -1,15 +0,0 @@ -docx==0.2.4 -python_docx==0.8.11 -langchain==0.0.123 -langdetect==1.0.9 -numpy==1.24.2 -openai==0.27.2 -pydub==0.25.1 -PyPDF4==1.27.0 -pytube==12.1.3 -streamlit==1.20.0 -streamlit_toggle_switch==1.0.2 -tiktoken==0.3.1 -requests==2.29.0 -youtube_transcript_api==0.6.0 - diff --git a/resources/prompt.json b/resources/prompt.json deleted file mode 100644 index 3be18b8..0000000 --- a/resources/prompt.json +++ /dev/null @@ -1,42 +0,0 @@ -[ - { - "type": "recursive", - "legacy": true, - "prompt": "Provide a detailed and comprehensive summary of the following content in flawless [LANGUAGE], ensuring all key points are covered. Create a markdown heading (###) that encapsulates the core information of the content. Make sure it is answered in [LANGUAGE].", - "variables": [ - { - "name": "[LANGUAGE]" - } - ] - }, - { - "type": "recursive", - "legacy": false, - "prompt": "Write a detailed and comprehensive explanation of the following in perfect [LANGUAGE] with no grammar issues, ensuring all key points are covered. Create a markdown heading (###) that encapsulates the core information:\n\n{text}\n\nStructured markdown summary with heading (###) in fluent [LANGUAGE]:", - "variables": [ - { - "name": "[LANGUAGE]" - } - ] - }, - { - "type": "final", - "legacy": true, - "prompt": "identify headings in the transcript and summarise them into five headings. Use #### headings in markdown. Under headings, summarise at least 3 key points and then provide detail explanation of the concept based on the following text in the way that can be read fluently, make sense and avoid repetition. Make sure to include all information. Write it in beautiful and structured markdown in perfect [LANGUAGE]. ", - "variables": [ - { - "name": "[LANGUAGE]" - } - ] - }, - { - "type": "final", - "legacy": false, - "prompt": "Write a detailed summary of the following in [LANGUAGE]:\n\n{text}\n\nIdentify and summarise them into five headings. Use #### headings in markdown. Under headings, summarize a list of key points that best encapsulate the core information. Structured markdown summary with headings in perfect [LANGUAGE] (####): ", - "variables": [ - { - "name": "[LANGUAGE]" - } - ] - } -] \ No newline at end of file diff --git a/shared_dependencies.md b/shared_dependencies.md new file mode 100755 index 0000000..5b4f3a4 --- /dev/null +++ b/shared_dependencies.md @@ -0,0 +1,17 @@ +Shared Dependencies: + +1. `ApplicationOverview`: Information about the application's purpose, audience, and technology stack used across multiple documents for context. +2. `DataModelSchemas`: Definitions of data structures used in Entities, which are referenced in Use Cases, Interface Adapters, and Testing Methods. +3. `BusinessLogicFunctions`: Function names and descriptions from the Core Business Logic that are called or referenced in Use Cases, Interface Adapters, and Testing Methods. +4. `UseCaseDescriptions`: Descriptions of business processes that are referenced in Interface Adapters and Testing Methods for integration and validation purposes. +5. `InterfaceComponentIDs`: ID names of DOM elements used in Interface Adapters that JavaScript functions will interact with, potentially referenced in Testing Methods for UI tests. +6. `APIIntegrationDetails`: Information about external APIs from Frameworks and Drivers that are used in Interface Adapters and may require security considerations. +7. `SecurityProtocolNames`: Names of security protocols from Security and Authentication that are implemented in Interface Adapters and Frameworks and Drivers. +8. `AuthenticationFunctionNames`: Names of authentication functions that are used in Security and Authentication, potentially referenced in Interface Adapters and Frameworks and Drivers. +9. `TestingMethodNames`: Names of testing methods from Testing and Quality Assurance that are used to validate Entities, Use Cases, and Interface Adapters. +10. `BuildAutomationScripts`: Script names and descriptions from Build Process and Dependency Management that are used to compile and deploy the application, potentially referenced in Testing Methods. +11. `SharedLibraryNames`: Names of shared libraries or dependencies from Inter-component Relationships that are used across various components of the application. +12. `DocumentationFileNames`: Names of the Markdown files that are consistently used across the Documentation Assembly and Organization to ensure a coherent structure. +13. `MaintenanceStrategies`: Strategies for maintaining documentation that are outlined in Documentation and Knowledge Sharing and applied in Documentation Assembly and Organization. + +These shared dependencies are the common elements that will be referenced or used across the generated Markdown files to ensure consistency and coherence in the documentation of the application's architecture and processes. \ No newline at end of file diff --git a/src/Components/Info.py b/src/Components/Info.py deleted file mode 100644 index 72f324f..0000000 --- a/src/Components/Info.py +++ /dev/null @@ -1,18 +0,0 @@ -import streamlit as st -import Modules.file_io as file_io - - -def info(): - info_panel = st.container() - - manifest = 'src/manifest.json' - st.session_state['MANIFEST'] = manifest_data = file_io.read_json(manifest) - - with info_panel: - st.markdown('---') - st.markdown(f"# {manifest_data['name']}") - st.markdown(f"Version: `{manifest_data['version']}`") - st.markdown(f"Author: {manifest_data['author']}") - st.markdown(f"[Report a bug]({manifest_data['bugs']['url']})") - st.markdown(f"[GitHub repo]({manifest_data['homepage']})") - st.markdown(f"License: [{manifest_data['license']['type']}]({manifest_data['license']['url']})") \ No newline at end of file diff --git a/src/Components/StreamlitSetup.py b/src/Components/StreamlitSetup.py deleted file mode 100644 index b57f3ce..0000000 --- a/src/Components/StreamlitSetup.py +++ /dev/null @@ -1,36 +0,0 @@ -import streamlit as st -import Data.caption_languages as data -import Modules.file_io as file_io - -def setup(): - st.set_page_config(page_title="SumGPT", page_icon="📝", layout="wide") - - if not st.session_state.get('OPENAI_API_KEY'): - st.session_state['OPENAI_API_KEY'] = None - - if not st.session_state.get('OPENAI_PERSONA_REC'): - st.session_state['OPENAI_PERSONA_REC'] = None - - if not st.session_state.get('OPENAI_PERSONA_SUM'): - st.session_state['OPENAI_PERSONA_SUM'] = None - - if not st.session_state.get('CHUNK_SIZE'): - st.session_state['CHUNK_SIZE'] = None - - if not st.session_state.get('OPENAI_PARAMS'): - st.session_state['OPENAI_PARAMS'] = None - - if not st.session_state.get('DELAY'): - st.session_state['DELAY'] = 0 - - if not st.session_state.get('FINAL_SUMMARY_MODE'): - st.session_state['FINAL_SUMMARY_MODE'] = False - - if not st.session_state.get('CAPTION_LANGUAGES'): - st.session_state['CAPTION_LANGUAGES'] = data.languages + data.auto_languages - - if not st.session_state.get('PREVIOUS_RESULTS'): - st.session_state['PREVIOUS_RESULTS'] = None - - if not st.session_state.get('MANIFEST'): - st.session_state["MANIFEST"] = file_io.read_json("src/manifest.json") \ No newline at end of file diff --git a/src/Components/__init__.py b/src/Components/__init__.py deleted file mode 100644 index 9391db9..0000000 --- a/src/Components/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from Components import sidebar -from Components import StreamlitSetup -from Components import Info -__all__ = ['sidebar', 'StreamlitSetup', 'Info'] \ No newline at end of file diff --git a/src/Components/sidebar.py b/src/Components/sidebar.py deleted file mode 100644 index b5b3d41..0000000 --- a/src/Components/sidebar.py +++ /dev/null @@ -1,187 +0,0 @@ -import streamlit as st -import GPT -import Modules.file_io as file_io -from streamlit_toggle import st_toggle_switch -import Components -from typing import Any, Dict, List, Tuple, Union -import json - - -def set_openai_api_key(api_key: str): - st.session_state["OPENAI_API_KEY"] = api_key - - -def set_openai_persona(persona_rec: str, persona_sum: str): - st.session_state["OPENAI_PERSONA_REC"] = persona_rec - st.session_state["OPENAI_PERSONA_SUM"] = persona_sum - - -def set_param(params: GPT.param): - st.session_state["OPENAI_PARAMS"] = params - - -def set_chunk_size(size: int): - st.session_state['CHUNK_SIZE'] = size - - -def set_delay(time: int): - st.session_state['DELAY'] = time - - -def set_final_summary_mode(mode: bool): - st.session_state['FINAL_SUMMARY_MODE'] = mode - - -def _set_config(config_file, key: str, default_value): - if config_file: - return file_io.read_json_upload(config_file, key) - else: - return default_value - -def _set_language(language: str): - st.session_state['OUTPUT_LANGUAGE'] = language - -def _set_legacy(enable: bool): - st.session_state['LEGACY'] = enable -def _legacy(enable: bool, legacy, experimental): - if not enable: - return experimental - else: - return legacy -def _extract_prompt(json_data: List[Dict[str,Union[bool, str]]], target_type: str, target_legacy: bool, language: str = "English") -> str | None: - for item in json_data: - if item["type"] == target_type and item["legacy"] == target_legacy: - prompt = item["prompt"] - new_prompt = prompt.replace("[LANGUAGE]", language) - return new_prompt - return None - -def sidebar(): - with st.sidebar: - st.markdown("## How to use\n" - "1. 🔑 Enter your [OpenAI API key](https://beta.openai.com/account/api-keys)\n" - "2. 📁 upload your file\n" - "3. 🏃 Run\n" - "---") - - config_file = st.file_uploader("📁 Import Configs", type=['json']) - - api_input = st.text_input(label="🔑 OpenAI API Key", - placeholder="Enter your OpenAI API key (sk-...)", - type="password", - help="You can get your API key from https://beta.openai.com/account/api-keys", - value=_set_config(config_file, "OPENAI_API_KEY", "")) - - enable_legacy = st_toggle_switch(label="Legacy", default_value=_set_config(config_file, "LEGACY", False)) - enable_final_summary = st_toggle_switch(label="Enable Final Summary", - default_value=_set_config(config_file, "FINAL_SUMMARY_MODE", False)) - if enable_final_summary: - set_final_summary_mode(True) - if st.session_state['FINAL_SUMMARY_MODE'] != enable_final_summary: - set_final_summary_mode(enable_final_summary) - - with st.expander('🤖 Bot Persona'): - language_options = ['English', 'Chinese', 'Japanese', 'Korean', 'Spanish', 'French', 'German'] - language_index = language_options.index(_set_config(config_file, "LANGUAGE", 'English')) - language = st.selectbox('Language', options=language_options, index=language_index) - _set_language(language) - - prompts = file_io.read_json("resources/prompt.json") - - persona_rec_legacy = _extract_prompt(prompts, "recursive", True, language) - persona_rec = _extract_prompt(prompts, "recursive", False, language) - persona_rec = st.text_area('Bot Persona Recursive', - value=_set_config(config_file, "OPENAI_PERSONA_REC", _legacy(enable_legacy, persona_rec_legacy, persona_rec)), - help='System message is a pre-defined message used to instruct the assistant at the ' - 'beginning of a conversation. iterating and ' - 'experimenting with potential improvements can help to generate better outputs.' - 'Make sure to use casual language.', - height=250) - if enable_final_summary: - persona_sum_legacy = _extract_prompt(prompts, "final", True, language) - persona_sum = _extract_prompt(prompts, "final", False, language) - - persona_sum = st.text_area('Bot Persona Total Sum', - value=_set_config(config_file, "OPENAI_PERSONA_SUM", _legacy(enable_legacy, persona_sum_legacy, persona_sum)), - help='This is a pre-defined message for total summarization that is used to' - 'instruct the assistant at the beginning of a conversation. ', - height=300) - else: - persona_sum = "" - - with st.expander('🔥 Advanced Options'): - model_options = ['gpt-3.5-turbo','gpt-3.5-turbo-16k', 'gpt-4'] - model_index = model_options.index(_set_config(config_file, "MODEL", 'gpt-3.5-turbo')) - model = st.selectbox("Model", options=model_options, index=model_index) - - if model == 'gpt-4': - max_chunk = 4000 - elif model == 'gpt-3.5-turbo-16k': - max_chunk = 16000 - else: - max_chunk = 2500 - chunk_size = st.slider('Chunk Size (word count)', min_value=0, max_value=max_chunk, step=20, - value=_set_config(config_file, "CHUNK_SIZE", 800)) - max_tokens_rec = st.slider('Max Tokens - Recursive Summary', min_value=0, max_value=4090, step=20, - value=_set_config(config_file, "MAX_TOKENS_REC", 250)) - if enable_final_summary: - max_tokens_final = st.slider('Max Tokens - Final Summary', min_value=0, max_value=4090, step=20, - value=_set_config(config_file, "MAX_TOKENS_FINAL", 650)) - else: - max_tokens_final = 0 - temperature = st.slider('Temperature', min_value=0.0, max_value=1.0, step=0.05, - value=_set_config(config_file, "TEMPERATURE", 0.7)) - top_p = st.slider('Top P', min_value=0.0, max_value=1.0, step=0.05, - value=_set_config(config_file, "TOP_P", 1.0)) - frequency_penalty = st.slider('Frequency Penalty', min_value=0.0, max_value=2.0, step=0.1, - value=_set_config(config_file, "FREQUENCY_PENALTY", 0.0)) - presence_penalty = st.slider('Presence Penalty', min_value=0.0, max_value=2.0, step=0.1, - value=_set_config(config_file, "PRESENCE_PENALTY", 0.0)) - if st_toggle_switch(label="Delay (free openAI API user)", - default_value=_set_config(config_file, "ENABLE_DELAY", False)): - delay = st.slider('Delay (seconds)', min_value=0, max_value=60, step=1, - value=_set_config(config_file, "DELAY_TIME", 1)) - else: - delay = 0 - param = GPT.param.gpt_param( - model=model, - max_tokens_final=max_tokens_final, - max_tokens_rec=max_tokens_rec, - temperature=temperature, - top_p=top_p, - frequency_penalty=frequency_penalty, - presence_penalty=presence_penalty - ) - - st.download_button(label="📥 Export Configs", - data=json.dumps({ - "OPENAI_API_KEY": api_input, - "FINAL_SUMMARY_MODE": enable_final_summary, - "OPENAI_PERSONA_REC": persona_rec, - "OPENAI_PERSONA_SUM": persona_sum, - "CHUNK_SIZE": chunk_size, - "MAX_TOKENS_REC": max_tokens_rec, - "MAX_TOKENS_FINAL": max_tokens_final, - "TEMPERATURE": temperature, - "TOP_P": top_p, - "FREQUENCY_PENALTY": frequency_penalty, - "PRESENCE_PENALTY": presence_penalty, - "MODEL": model, - "ENABLE_DELAY": delay > 0, - "DELAY_TIME": delay, - "LANGUAGE": language, - "LEGACY": enable_legacy - }, indent=4), - file_name="configs.json") - Components.Info.info() - - if api_input: - set_openai_api_key(api_input) - - if persona_rec: - set_openai_persona(persona_rec, persona_sum) - - set_chunk_size(chunk_size) - set_param(param) - set_delay(delay) - _set_legacy(enable_legacy) \ No newline at end of file diff --git a/src/Data/__init__.py b/src/Data/__init__.py deleted file mode 100644 index 4de9124..0000000 --- a/src/Data/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from Data import caption_languages - -__all__ = ['caption_languages'] \ No newline at end of file diff --git a/src/Data/caption_languages.py b/src/Data/caption_languages.py deleted file mode 100644 index acec65e..0000000 --- a/src/Data/caption_languages.py +++ /dev/null @@ -1,6 +0,0 @@ -languages = [ - 'en', 'es', 'fr', 'de', 'it', 'pt', 'ru', 'ja', 'ko', 'zh-Hans', 'zh-Hant', 'zh-TW', 'zh-CN', 'zh', 'ar', 'hi', 'th' -] - -auto_languages = ['a.' + _language for _language in languages] - diff --git a/src/GPT/__init__.py b/src/GPT/__init__.py deleted file mode 100644 index 0bcd76d..0000000 --- a/src/GPT/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from GPT import misc -from GPT import embeddings -from GPT import bot -from GPT import param -from GPT import generate - -__all__ = ['misc', 'embeddings', 'bot', 'param', 'generate'] \ No newline at end of file diff --git a/src/GPT/bot.py b/src/GPT/bot.py deleted file mode 100644 index ce8dd86..0000000 --- a/src/GPT/bot.py +++ /dev/null @@ -1,48 +0,0 @@ -import openai -from typing import Any, Dict, List, Tuple, Union - - -class OpenAIChatBot: - """A class to interact with the OpenAI API.""" - - def __init__(self, api_key: str, persona: str, model: str, max_tokens: int, temperature: float, top_p: float, - frequency_penalty: float, presence_penalty: float): - openai.api_key = api_key - self.persona = persona - self.model = model - self.max_tokens = max_tokens - self.temperature = temperature - self.top_p = top_p - self.frequency_penalty = frequency_penalty - self.presence_penalty = presence_penalty - - def chat_stream(self, prompt: str) -> openai.api_resources.chat_completion.ChatCompletion: - """Returns the streamed response from the OpenAI API.""" - completions = openai.ChatCompletion.create( - model=self.model, - max_tokens=self.max_tokens, - temperature=self.temperature, - top_p=self.top_p, - frequency_penalty=self.frequency_penalty, - presence_penalty=self.presence_penalty, - stream=True, - messages=[ - {"role": "system", "content": self.persona}, - {"role": "user", "content": prompt} - ]) - return completions - - def chat(self, prompt: str) -> Tuple[str, str]: - """Returns the response from the OpenAI API.""" - completions = openai.ChatCompletion.create( - model=self.model, - max_tokens=self.max_tokens, - temperature=self.temperature, - top_p=self.top_p, - frequency_penalty=self.frequency_penalty, - presence_penalty=self.presence_penalty, - messages=[ - {"role": "system", "content": self.persona}, - {"role": "user", "content": f"{self.persona} '{prompt}'"} - ]) - return completions['choices'][0]['message']['content'], completions['choices'][0]['finish_reason'] diff --git a/src/GPT/embeddings.py b/src/GPT/embeddings.py deleted file mode 100644 index 3e6cb50..0000000 --- a/src/GPT/embeddings.py +++ /dev/null @@ -1,12 +0,0 @@ -import openai - - -class openAIEmbeddings: - def __init__(self, api_key: str): - openai.api_key = api_key - - def embedding(self, content: str, engine: str = 'text-embedding-ada-002') -> float: - """Returns the embedding vector of a string.""" - response = openai.Embedding.create(input=content, engine=engine) - vector = response['data'][0]['embedding'] - return vector diff --git a/src/GPT/generate.py b/src/GPT/generate.py deleted file mode 100644 index 627faf6..0000000 --- a/src/GPT/generate.py +++ /dev/null @@ -1,52 +0,0 @@ -import GPT.bot -import streamlit as st -import GPT.param -from typing import Any, Dict, List, Tuple, Union - - -def get_answer_stream(content: str): - """Returns a stream of responses from the OpenAI API.""" - params = st.session_state["OPENAI_PARAMS"] - previous_char = '' - bot = GPT.bot.OpenAIChatBot(st.session_state["OPENAI_API_KEY"], - st.session_state["OPENAI_PERSONA"], - params.model, - params.max_tokens_rec, - params.temperature, - params.top_p, - params.frequency_penalty, - params.presence_penalty) - responses = bot.chat_stream(content) - response_panel = st.empty() - for response_json in responses: - choice = response_json['choices'][0] - if choice['finish_reason'] == 'stop': - break - - # error handling - if choice['finish_reason'] == 'length': - st.warning('⚠️Result cut off due to length. Consider increasing the max tokens parameter.') - break - - delta = choice['delta'] - if 'role' in delta or delta == {}: - char = '' - else: - char = delta['content'] - answer = previous_char + char - response_panel.info(answer) - - -def get_answer(content: str, max_tokens, persona: str) -> Tuple[str, str]: - """Returns a response from the OpenAI API.""" - params = st.session_state["OPENAI_PARAMS"] - bot = GPT.bot.OpenAIChatBot(st.session_state["OPENAI_API_KEY"], - persona, - params.model, - max_tokens, - params.temperature, - params.top_p, - params.frequency_penalty, - params.presence_penalty) - response, finish_reason = bot.chat(content) - return response, finish_reason diff --git a/src/GPT/misc.py b/src/GPT/misc.py deleted file mode 100644 index b93481c..0000000 --- a/src/GPT/misc.py +++ /dev/null @@ -1,98 +0,0 @@ -import openai -from langchain.llms import OpenAI -import os -import streamlit as st -from typing import Any, Dict, List, Tuple, Union - - -def validate_api_key(api_key: str) -> bool: - """Validates the OpenAI API key by trying to create a completion.""" - openai.api_key = api_key - try: - openai.ChatCompletion.create( - model="gpt-3.5-turbo", - max_tokens=1, - messages=[ - {"role": "user", "content": "Hello!"} - ] - ) - return True - except openai.error.AuthenticationError: - return False - - -def predict_token(param, chunks) -> Dict[str, int]: - """predict how many tokens to generate.""" - if st.session_state["OPENAI_API_KEY"] is not None: - os.environ['OPENAI_API_KEY'] = st.session_state["OPENAI_API_KEY"] - llm = OpenAI() - prompt_token_total = 0 - completion_token_total = 0 - for chunk in chunks: - prompt_token = llm.get_num_tokens(chunk['content']) - prompt_token_total += prompt_token - completion_token_total += param.max_tokens_rec - - if st.session_state['FINAL_SUMMARY_MODE']: - completion_token_total += param.max_tokens_final - total_token = prompt_token_total + completion_token_total - token = {'total': total_token, - 'prompt': prompt_token_total, - 'completion': completion_token_total} - - return token - else: - return {'total': 0, 'prompt': 0, 'completion': 0} - - -def predict_token_single(chunk: Dict[str, Union[str, float]] | str, max_tokens: int = None) -> int: - """predict how many tokens to generate.""" - if st.session_state["OPENAI_API_KEY"] is not None: - os.environ['OPENAI_API_KEY'] = st.session_state["OPENAI_API_KEY"] - llm = OpenAI() - if isinstance(chunk, str): - chunk_content = chunk - else: - chunk_content = chunk['content'] - chunk_token = llm.get_num_tokens(chunk_content) - if max_tokens is not None: - chunk_token += max_tokens - - return chunk_token - else: - return 0 - - -def is_tokens_exceeded(param, chunks, max_token: int = 4096) -> Dict[str, Union[bool, str]]: - """Checks if the number of tokens used has exceeded the limit.""" - - # check recursive chunks tokens - rec_chunks_token = [] - for chunk in chunks: - chunk_token = predict_token_single(chunk, param.max_tokens_rec) - rec_chunks_token.append(chunk_token) - - - # check final chunks tokens - final_prompt_token = len(chunks) * param.max_tokens_rec - final_completion_token = param.max_tokens_final - final_chunks_token = final_prompt_token + final_completion_token - - # evaluate - if max(rec_chunks_token) > max_token: - return {'exceeded': True, - 'reason': 'recursive', - 'message': f"**[ Recursive summary ]** tokens exceeded. Max tokens allowed: {max_token}. Tokens used: {max(rec_chunks_token)}\n" - f"(Prompt: {max(rec_chunks_token) - param.max_tokens_rec}, " - f"Completion: {param.max_tokens_rec})"} - - elif final_chunks_token > max_token and st.session_state['FINAL_SUMMARY_MODE']: - return {'exceeded': True, - 'reason': 'final', - 'message': f"**[ Final summary ]** tokens exceeded. Max tokens allowed: {max_token}. Tokens used: {final_chunks_token}\n" - f"(Prompt: {final_prompt_token}, Completion: {final_completion_token})"} - - else: - return {'exceeded': False, - 'reason': '', - 'message': ''} diff --git a/src/GPT/param.py b/src/GPT/param.py deleted file mode 100644 index 866f112..0000000 --- a/src/GPT/param.py +++ /dev/null @@ -1,11 +0,0 @@ - -class gpt_param: - def __init__(self, model: str, max_tokens_final: int, max_tokens_rec: int, temperature: float, top_p: float, - frequency_penalty: float, presence_penalty: float): - self.model = model - self.max_tokens_rec = max_tokens_rec - self.max_tokens_final = max_tokens_final - self.temperature = temperature - self.top_p = top_p - self.frequency_penalty = frequency_penalty - self.presence_penalty = presence_penalty diff --git a/src/Modules/Youtube.py b/src/Modules/Youtube.py deleted file mode 100644 index f399cb3..0000000 --- a/src/Modules/Youtube.py +++ /dev/null @@ -1,97 +0,0 @@ -import requests -import re -from youtube_transcript_api import YouTubeTranscriptApi, TranscriptsDisabled, NoTranscriptFound -import streamlit as st -from typing import Any, Dict, List, Tuple, Union - - -manifest = st.session_state["MANIFEST"] -def _error_report_msg(youtube_url): - return f"Please create an issue on [GitHub]({manifest['bugs']['url']}). " \ - f"Please include the YouTube URL ({youtube_url}), version number ({manifest['version']}) " \ - f"and all necessary information to replicate the error. " \ - f"**Before creating a new issue, please check if the problem has already been reported.**" - -def _extract_video_id_from_url(url): - video_id_pattern = r'(?:v=|/v/|youtu\.be/|/embed/|/e/)([^?&"\'>]+)' - match = re.search(video_id_pattern, url) - if match: - return match.group(1) - else: - raise ValueError("Invalid YouTube URL") - -def get_video_title(youtube_url): - video_id = _extract_video_id_from_url(youtube_url) - url = f'https://www.youtube.com/watch?v={video_id}' - response = requests.get(url) - title_pattern = r'(.+?) - YouTube<\/title>' - match = re.search(title_pattern, response.text) - if match: - title = match.group(1) - return title - else: - return None - -def get_available_subtitle_languages(video_id): - try: - transcript_list = YouTubeTranscriptApi.list_transcripts(video_id) - languages = [transcript.language_code for transcript in transcript_list] - return languages - except Exception as e: - print(f"Error fetching available subtitle languages: {e}") - return [] - -def get_video_captions(youtube_url, languages): - video_id = _extract_video_id_from_url(youtube_url) - simplified_url = f'https://www.youtube.com/watch?v={video_id}' - - available_language = get_available_subtitle_languages(video_id) - - if not any(lang in languages for lang in available_language) and available_language != []: - print(f"Failed to retrieve transcript: Language {available_language} is/are not yet supported for {simplified_url}.") - st.error(f'❌ Language {available_language} is/are not yet supported for {simplified_url}.\n\n' + _error_report_msg(simplified_url)) - st.stop() - - for language in languages: - try: - transcript = YouTubeTranscriptApi.get_transcript(video_id, languages=[language]) - captions = "" - for item in transcript: - captions += item['text'] + "\n" - return captions - - except NoTranscriptFound as e: - if language == languages[-1]: - print(f"Language {available_language} exist in language list but failed to retrieve in YouTubeTranscriptApi.get_transcript: {e}") - st.error(f'❌ Language {available_language} exist in language list but failed to retrieve in `YouTubeTranscriptApi.get_transcript`:\n\n' - f'languages = {available_language}\n\n' - f'language list = {languages}\n\n' - + _error_report_msg(simplified_url)) - st.stop() - else: - continue - - except TranscriptsDisabled: - print(f"Failed to retrieve transcript: transcripts disabled for {simplified_url}") - st.error(f'❌ Subtitles not available for {simplified_url}! \n\n---' - f'\n**Instruction:**\n\n' - f'1. Verify if the [video]({simplified_url}) has subtitles available.\n\n' - f"2. If you are confident that subtitles are available in the video but could not be retrieved, " - + _error_report_msg(simplified_url)) - st.stop() - raise TranscriptsDisabled - - except Exception as e: - print(e) - st.error(f'❌ Failed to fetch data from YouTube for {simplified_url}. \n\n' - f'{_error_report_msg(simplified_url)}' - f'\n\nError: \n\n---\n\n{e}') - st.stop() - break - -@st.cache_data(show_spinner=False) -def extract_youtube_transcript(url: str, lang_code: str | List[str] = 'a.en') -> Tuple[str, str]: - """Extracts the transcript from a YouTube video.""" - transcript = get_video_captions(url, lang_code) - title = get_video_title(url) - return transcript, title diff --git a/src/Modules/__init__.py b/src/Modules/__init__.py deleted file mode 100644 index 412ace4..0000000 --- a/src/Modules/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from Modules import file_io - -__all__ = ['file_io'] \ No newline at end of file diff --git a/src/Modules/file_io.py b/src/Modules/file_io.py deleted file mode 100644 index 0f214e8..0000000 --- a/src/Modules/file_io.py +++ /dev/null @@ -1,99 +0,0 @@ -import re -import PyPDF4 -import docx -from typing import Any, Dict, List, Tuple, Union -from pydub import AudioSegment -import math -import json -import streamlit as st - - - -@st.cache_data() -def read_json(file, key: str = None) -> Any: - """Reads a json file and returns the value of a key.""" - with open(file, "r") as f: - data = json.load(f) - if key and isinstance(data, dict): - return data[key] - elif key and isinstance(data, list): - return [d[key] for d in data] - else: - return data - - -@st.cache_data() -def read_json_upload(file, key: str) -> Any: - """Reads a json file and returns the value of a key.""" - if not isinstance(file, str): - f = file.getvalue().decode("utf-8") - data = json.loads(f) - return data[key] - - -@st.cache_data() -def read_txt(file, encoding: str = "utf-8") -> str: - """Reads a text file.""" - return file.read().decode(encoding) - - -@st.cache_data() -def read_pdf(file) -> List[str]: - """Reads a pdf file.""" - pdfReader = PyPDF4.PdfFileReader(file, strict=False) - texts = [] - for page in range(pdfReader.numPages): - text = pdfReader.getPage(page).extractText() - # Merge hyphenated words - text = re.sub(r"(\w+)-\n(\w+)", r"\1\2", text) - # Fix newlines in the middle of sentences - text = re.sub(r"(?<!\n\s)\n(?!\s\n)", " ", text.strip()) - # Remove multiple newlines - text = re.sub(r"\n\s*\n", "\n\n", text) - - texts.append(text) - return texts - - -@st.cache_data() -def read_docx(file) -> str: - """Reads a docx file.""" - doc = docx.Document(file) - text = "" - for para in doc.paragraphs: - # Remove multiple newlines - t = re.sub(r"\n\s*\n", "\n\n", para.text) - text += t + "\n" - return text - - -@st.cache_data() -def _split_audio(audio, chunk_size=2) -> List[AudioSegment]: - """Split audio into chunks of 10 minutes.""" - # load audio - audio = AudioSegment.from_file(audio, format="mp3") - # Define the chunk size (10 minutes default) - chunk_size = chunk_size * 60 * 1000 - # calculate the number of chunks - num_chunks = math.ceil(len(audio) / chunk_size) - chunks = [] - # split audio into chunks - for i in range(num_chunks): - start = i * chunk_size - end = start + chunk_size - chunk = audio[start:end] - chunks.append(chunk) - return chunks - - -@st.cache_data() -def read(file) -> str | List[str]: - """Reads a file and returns the content.""" - if file.name.endswith(".txt") or file.name.endswith(".md"): - return read_txt(file) - elif file.name.endswith(".pdf"): - return read_pdf(file) - elif file.name.endswith(".docx"): - return read_docx(file) - else: - raise ValueError("File type not supported") diff --git a/src/SumGPT.py b/src/SumGPT.py deleted file mode 100644 index b2040ea..0000000 --- a/src/SumGPT.py +++ /dev/null @@ -1,152 +0,0 @@ -import asyncio -import streamlit as st - -import Components.StreamlitSetup as StreamlitSetup -StreamlitSetup.setup() - -import Modules.Youtube -from Components.sidebar import sidebar -import Modules.file_io as file_io -import GPT -import util -import time - -app_header = st.container() - -file_handler = st.container() -content_handler = st.container() -result_handler = st.container() - -with app_header: - st.title("📝 SumGPT") - st.markdown("##### Summarize your text with OpenAI's GPT-3.5 / GPT-4 API") - st.markdown("##### [GitHub repo](https://github.com/sean1832/SumGPT)") - st.warning("🚧️ This app is still in beta. Please [report any bugs](https://github.com/sean1832/SumGPT/issues) to the GitHub repo.") - -sidebar() - -with file_handler: - if st.button("🔃 Refresh"): - st.cache_data.clear() - youtube_link_empty = st.empty() - upload_file_emtpy = st.empty() - - youtube_link = youtube_link_empty.text_input(label="🔗 YouTube Link", - placeholder="Enter your YouTube link", - help="Enter your YouTube link to download the video and extract the audio") - upload_file = upload_file_emtpy.file_uploader("📁 Upload your file", type=['txt', 'pdf', 'docx', 'md']) - if youtube_link: - upload_file_emtpy.empty() - with st.spinner("🔍 Extracting transcript..."): - transcript, title = Modules.Youtube.extract_youtube_transcript(youtube_link, st.session_state['CAPTION_LANGUAGES']) - file_content = {'name': f"{title}.txt", 'content': transcript} - elif upload_file: - youtube_link_empty.empty() - with st.spinner("🔍 Reading file... (mp3 file might take a while)"): - file_content = {'name': upload_file.name, 'content': file_io.read(upload_file)} - elif youtube_link and upload_file: - st.warning("Please only upload one file at a time") - else: - file_content = None - -with content_handler: - if file_content: - with st.expander("File Preview"): - if file_content['name'].endswith(".pdf"): - content = "\n\n".join(file_content['content']) - st.text_area(file_content['name'], content, height=200) - else: - content = file_content['content'] - st.text_area(file_content['name'], content, height=200) - -with result_handler: - if file_content: - chunks = [] - content = file_content['content'] - if file_content['name'].endswith(".pdf"): - content = "\n\n".join(file_content['content']) - chunks.extend(util.convert_to_chunks(content, chunk_size=st.session_state['CHUNK_SIZE'])) - - with st.expander(f"Chunks ({len(chunks)})"): - for chunk in chunks: - st.write(chunk) - - token_usage = GPT.misc.predict_token(st.session_state['OPENAI_PARAMS'], chunks) - param = st.session_state["OPENAI_PARAMS"] - prompt_token = token_usage['prompt'] - completion_token = token_usage['completion'] - if param.model == 'gpt-4': - price = round(prompt_token * 0.00003 + completion_token * 0.00006, 5) - st.markdown('**Note:** To access GPT-4, please [join the waitlist](https://openai.com/waitlist/gpt-4-api)' - " if you haven't already received an invitation from OpenAI.") - st.info("ℹ️️ Please keep in mind that GPT-4 is significantly **[more expensive](https://openai.com/pricing#language-models)** than GPT-3.5. ") - elif param.model == 'gpt-3.5-turbo-16k': - price = round(prompt_token * 0.000003 + completion_token *0.000004, 5) - else: - price = round(prompt_token * 0.0000015 + completion_token * 0.000002 , 5) - st.markdown( - f"Price Prediction: `${price}` || Total Prompt: `{prompt_token}`, Total Completion: `{completion_token}`") - # max tokens exceeded warning - exceeded = util.exceeded_token_handler(param=st.session_state['OPENAI_PARAMS'], chunks=chunks) - - # load cached results - if st.session_state['PREVIOUS_RESULTS'] is not None: - rec_responses = st.session_state['PREVIOUS_RESULTS']['rec_responses'] - rec_id = st.session_state['PREVIOUS_RESULTS']['rec_ids'] - final_response = st.session_state['PREVIOUS_RESULTS']['final_response'] - finish_reason_rec = st.session_state['PREVIOUS_RESULTS']['finish_reason_rec'] - finish_reason_final = st.session_state['PREVIOUS_RESULTS']['finish_reason_final'] - else: - rec_responses = None - rec_id = None - final_response = None - finish_reason_rec = None - finish_reason_final = None - - # finish_reason_rec = None - if st.button("🚀 Run", disabled=exceeded): - start_time = time.time() - st.cache_data.clear() - API_KEY = st.session_state['OPENAI_API_KEY'] - if not API_KEY and not GPT.misc.validate_api_key(API_KEY): - st.error("❌ Please enter a valid [OpenAI API key](https://beta.openai.com/account/api-keys).") - else: - with st.spinner("Summarizing... (this might take a while)"): - if st.session_state['LEGACY']: - rec_max_token = st.session_state['OPENAI_PARAMS'].max_tokens_rec - rec_responses, finish_reason_rec = util.recursive_summarize(chunks, rec_max_token) - if st.session_state['FINAL_SUMMARY_MODE']: - final_response, finish_reason_final = util.summarize(rec_responses) - else: - final_response = None - else: - completions, final_response = asyncio.run(util.summarize_experimental_concurrently(content, st.session_state['CHUNK_SIZE'])) - rec_responses = [d["content"] for d in completions] - rec_ids = [d["chunk_id"] for d in completions] - # save previous completions - resp = {'rec_responses': rec_responses, - 'rec_ids': rec_ids, - 'final_response': final_response, - 'finish_reason_rec': finish_reason_rec, - 'finish_reason_final': finish_reason_final} - if resp != st.session_state['PREVIOUS_RESULTS']: - st.session_state['PREVIOUS_RESULTS'] = resp - - end_time = time.time() - st.markdown(f"⏱️ Time taken: `{round(end_time - start_time, 2)}s`") - - if rec_responses is not None: - with st.expander("Recursive Summaries", expanded=not st.session_state['FINAL_SUMMARY_MODE']): - for i, response in enumerate(rec_responses): - st.info(f'{response}') - if finish_reason_rec == 'length': - st.warning('⚠️Result cut off due to length. Consider increasing the [Max Tokens Chunks] parameter.') - - if final_response is not None: - st.header("📝Summary") - st.info(final_response) - if finish_reason_final == 'length': - st.warning( - '⚠️Result cut off due to length. Consider increasing the [Max Tokens Summary] parameter.') - if final_response is not None or rec_responses is not None: - util.download_results(rec_responses, final_response) diff --git a/src/manifest.json b/src/manifest.json deleted file mode 100644 index 731522c..0000000 --- a/src/manifest.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "SumGPT", - "version": "1.0.8", - "license": { - "type": "MIT", - "url": "https://github.com/sean1832/SumGPT/blob/master/LICENSE" - }, - "author": "Zeke Zhang", - "homepage": "https://github.com/sean1832/SumGPT", - "repository": { - "type": "git", - "url": "https://github.com/sean1832/SumGPT" - }, - "bugs": { - "url": "https://github.com/sean1832/SumGPT/issues" - } -} \ No newline at end of file diff --git a/src/util.py b/src/util.py deleted file mode 100644 index 6fd2618..0000000 --- a/src/util.py +++ /dev/null @@ -1,237 +0,0 @@ -import os -import asyncio - -import numpy as np -from typing import Any, Dict, List, Tuple, Union - -from GPT.embeddings import openAIEmbeddings -import streamlit as st -import re -import GPT -import textwrap -from langdetect import detect -import time -from datetime import datetime - -from langchain.chat_models import ChatOpenAI -from langchain.docstore.document import Document -from langchain.prompts import PromptTemplate -from langchain.chains.summarize import load_summarize_chain -from langchain.chains import LLMChain - -def _similarity(v1, v2) -> np.ndarray: - """Returns the cosine similarity between two vectors.""" - return np.dot(v1, v2) - -@st.cache_data(show_spinner=False) -def _chunk_spliter(content: str, chunk_size: int = 1000, lang_base: str = 'latin') -> List[str]: - """Splits a string into chunks of a given size.""" - - sentences = re.split(r'(?<=[.?!,。，、！？·])\s+', content) - if lang_base == 'latin': - chunks = [] - chunk = '' - word_count = 0 - for sentence in sentences: - sentence += ' ' # add space at end to compensate for split - words = sentence.split() - sentence_word_count = len(words) - if word_count + sentence_word_count <= chunk_size: - chunk += sentence - word_count += sentence_word_count - else: - chunks.append(chunk.strip()) - chunk = sentence - word_count = sentence_word_count - # add the last chunk - if chunk: - chunks.append(chunk.strip()) - - new_chunks = [] - for c in chunks: - if c == '': - continue - if len(c.split()) > chunk_size + 25: - words = c.split() - small_chunks = [] - for i in range(0, len(words), chunk_size): - small_chunks.append(' '.join(words[i:i + chunk_size])) - new_chunks.extend(small_chunks) - else: - new_chunks.append(c) - return new_chunks - - else: - chunks = textwrap.wrap(content, width=chunk_size) - return chunks - - -def language_base(string: str) -> str: - try: - lang_code = detect(string) - latin_based = ['en', 'fr-ca', 'es'] - east_asian_based = ['zh', 'ja', 'ko'] - for lang in latin_based: - if lang_code.startswith(lang): - return 'latin' - for lang in east_asian_based: - if lang_code.startswith(lang): - return 'east_asian' - return 'other' - except KeyError: - return 'other' - -@st.cache_data(show_spinner=False) -def convert_to_chunks(content: str, chunk_size: int = 1000, enable_embedding: bool = False) -> List[Dict[str, float]]: - """Converts a string into chunks of a given size.""" - chunks_text = _chunk_spliter(content, chunk_size, language_base(content)) - chunks = [] - for i, chunk in enumerate(chunks_text): - if enable_embedding: - embedding = openAIEmbeddings(st.session_state["OPENAI_API_KEY"]) - chunks.append({'content': chunk, 'vector': embedding.embedding(chunk)}) - else: - chunks.append({'content': chunk, 'language_based': language_base(chunk), 'chunk_id': i}) - return chunks - - -def search_chunks(query: str, chunks: List[Dict[str, float]], count: int = 1) -> List[Dict[str, np.ndarray]]: - """Returns the top `count` chunks that are most similar to the query.""" - embedding = openAIEmbeddings(st.session_state["OPENAI_API_KEY"]) - vectors = embedding.embedding(query) - points = [] - - for chunk in chunks: - point = _similarity(vectors, chunk['vector']) - points.append({'content': chunk['content'], 'point': point}) - - # sort the points in descending order - ordered = sorted(points, key=lambda x: x['point'], reverse=True) - return ordered[0:count] - -@st.cache_data(show_spinner=False) -def convert_to_docs(chunks: List[Dict[str, Union[str, float]]]) -> List[Document] | Document: - """Converts a list of chunks into a list of documents.""" - docs = [] - for chunk in chunks: - content = chunk['content'] - metadata = {'chunk_id': chunk['chunk_id']} - doc = Document(page_content=content, metadata=metadata) - docs.append(doc) - return docs - -async def async_generate(chain, chunk)-> Dict[str, Union[str, int]]: - """Generates a summary asynchronously.""" - resp = await chain.arun(text=chunk['content']) - return {'content': resp, 'chunk_id': chunk['chunk_id']} - -async def summarize_experimental_concurrently(content: str, chunk_size: int = 1000) -> Tuple[List[Dict[str, Union[str, int]]], str]: - """Summarizes a string asynchronously.""" - os.environ['OPENAI_API_KEY'] = st.session_state["OPENAI_API_KEY"] - params = st.session_state['OPENAI_PARAMS'] - llm_rec = ChatOpenAI(model_name=params.model, - max_tokens=params.max_tokens_rec, - temperature=params.temperature, - top_p=params.top_p, - frequency_penalty=params.frequency_penalty, - presence_penalty=params.presence_penalty) - llm_final = ChatOpenAI(model_name=params.model, - max_tokens=params.max_tokens_final, - temperature=params.temperature, - top_p=params.top_p, - frequency_penalty=params.frequency_penalty, - presence_penalty=params.presence_penalty) - chunks = convert_to_chunks(content, chunk_size) - - REC_PROMPT = PromptTemplate(template=st.session_state['OPENAI_PERSONA_REC'], input_variables=['text']) - chain = LLMChain(llm=llm_rec, prompt=REC_PROMPT) - - tasks = [] - for chunk in chunks: - task = async_generate(chain, chunk) - tasks.append(task) - - outputs_rec = [] - progress_bar = st.progress(0, f"Generating summary 0/{len(chunks)}") - count = 1 - for coro in asyncio.as_completed(tasks): - output_rec = await coro - outputs_rec.append(output_rec) - progress_bar.progress(count / len(chunks), f"Generating summary {count}/{len(chunks)}") - count += 1 - rec_result = sorted(outputs_rec, key=lambda x: x['chunk_id']) - if st.session_state['FINAL_SUMMARY_MODE']: - FINAL_PROMPT = PromptTemplate(template=st.session_state['OPENAI_PERSONA_SUM'], input_variables=['text']) - chain = load_summarize_chain(llm_final, chain_type='stuff', prompt=FINAL_PROMPT) - docs = convert_to_docs(rec_result) - final_result = chain.run(docs) - else: - final_result = None - return rec_result, final_result - -@st.cache_data(show_spinner=False) -def recursive_summarize(chunks: List[Dict[str, Union[str, float]]], max_tokens) -> Tuple[List[str], str]: - """Returns a recursive summary of the given content.""" - recursiveSumTexts = [] - finish_reason = '' - chunks_length = len(chunks) - count = 0 - progress_bar = st.progress(0) - for chunk in chunks: - content = chunk['content'] - text, finish_reason = GPT.generate.get_answer(content, - max_tokens=max_tokens, - persona=st.session_state['OPENAI_PERSONA_REC']) - recursiveSumTexts.append(text) - progress_bar.progress((count + 1) / chunks_length) - count += 1 - time.sleep(st.session_state['DELAY']) - - return recursiveSumTexts, finish_reason - - -@st.cache_data(show_spinner=False) -def summarize(message: List[str] | str) -> Tuple[str, str]: - """Returns a summary of the given content.""" - if isinstance(message, list): - join_msg = ' '.join(message) - else: - join_msg = message - - params = st.session_state['OPENAI_PARAMS'] - max_asw_tokens_final = params.max_tokens_final - - answer, finish_reason = GPT.generate.get_answer(join_msg, max_tokens=max_asw_tokens_final, - persona=st.session_state['OPENAI_PERSONA_SUM']) - return answer, finish_reason - - -def download_results(rec_responses, final_response): - """Downloads the results as a txt file.""" - joint_rec_response = f"=====recursive responses=====\n\n" + '\n\n'.join(rec_responses) - joint_final_response = f"{joint_rec_response}\n\n======final response=====\n\n{final_response}" - now = datetime.now() - if final_response is not None: - st.download_button("📥 Download Summary", - joint_final_response, - file_name=f"summary_{now.strftime('%Y-%m-%d_%H-%M')}.md") - else: - st.download_button("📥 Download Summary", - joint_rec_response, - file_name=f"summary_{now.strftime('%Y-%m-%d_%H-%M')}.md") - - -def exceeded_token_handler(param, chunks) -> bool: - """Handles the case where the user has exceeded the number of tokens.""" - if param.model == 'gpt-4': - max_token = 8100 - elif param.model == 'gpt-3.5-turbo-16k': - max_token = 16385 - else: - max_token = 4096 - info = GPT.misc.is_tokens_exceeded(param, chunks, max_token) - if info['exceeded']: - st.error(f"❌ {info['message']}") - return True - else: - return False diff --git a/tools/get-requirement.bat b/tools/get-requirement.bat deleted file mode 100644 index a5c88e2..0000000 --- a/tools/get-requirement.bat +++ /dev/null @@ -1,16 +0,0 @@ -@echo off -cd.. -echo Activating Virtural environment... -call .\venv\Scripts\activate - -echo upgrading pip... -python -m pip install --upgrade pip - - -echo Installing pipreqs... -pip install pipreqs - -echo Export to requirements.txt -pipreqs . --force --encoding utf-8 - -pause \ No newline at end of file