diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml index 9307cc79e4..2488cd8a52 100644 --- a/ansible/inventory/env/group_vars/all.yml +++ b/ansible/inventory/env/group_vars/all.yml @@ -108,7 +108,7 @@ cert_rc_base_url: "http://{{private_ingressgateway_ip}}/registry-service/api/v1" cert_rc_entity: "TrainingCertificate" certificate_base_path: "https://{{domain_name}}/certs" cert_domain_url: "{{proto}}://{{domain_name}}" -cert_container_name: "{{cert_env_prefix}}-e-credentials" +cert_container_name: "e-credentials" cert_cloud_storage_type: "azure" cert_azure_storage_secret: "{{sunbird_private_storage_account_key}}" cert_azure_storage_key: "{{sunbird_private_storage_account_name}}" diff --git a/ansible/roles/learning-service/templates/application.conf.j2 b/ansible/roles/learning-service/templates/application.conf.j2 index 11f45d05b7..8d93d60539 100644 --- a/ansible/roles/learning-service/templates/application.conf.j2 +++ b/ansible/roles/learning-service/templates/application.conf.j2 @@ -196,9 +196,7 @@ job.request.event.mimetype=["application/pdf", "image/x-quicktime", "video/avi", "video/mpeg", - "video/quicktime", "video/3gpp", - "video/mp4", "video/ogg", "video/webm", "video/msvideo", @@ -209,7 +207,6 @@ job.request.event.mimetype=["application/pdf", "audio/mp4", "audio/mpeg", "audio/ogg", - "audio/webm", "audio/x-wav", "audio/wav", "audio/mpeg3", @@ -253,12 +250,10 @@ cloud_storage_type="{{ cloud_store }}" azure_storage_key="{{sunbird_public_storage_account_name}}" azure_storage_secret="{{sunbird_public_storage_account_key}}" azure_storage_container="{{ azure_public_container }}" - aws_storage_key = "{{ aws_storage_key }}" aws_storage_secret: "{{ aws_storage_secret }}" aws_storage_container: "{{ aws_storage_container }}" - installation.id="{{ instance_name }}" # Configuration for DIALCode linking diff --git a/ansible/roles/lp-contenttool/templates/application.conf.j2 b/ansible/roles/lp-contenttool/templates/application.conf.j2 index 7444400ad2..97096a79e9 100644 --- a/ansible/roles/lp-contenttool/templates/application.conf.j2 +++ b/ansible/roles/lp-contenttool/templates/application.conf.j2 @@ -10,6 +10,9 @@ destination.storage_type="{{ contenttool.destination.storage_type }}" azure_storage_key="{{sunbird_public_storage_account_name}}" azure_storage_secret="{{sunbird_public_storage_account_key}}" azure_storage_container="{{ azure_public_container }}" +aws_storage_key = "{{ aws_storage_key }}" +aws_storage_secret: "{{ aws_storage_secret }}" +aws_storage_container: "{{ aws_storage_container }}" content.external_fields="body,stageIcons,screenshots" content.extract_mimetype="application/vnd.ekstep.h5p-archive,application/vnd.ekstep.ecml-archive,application/vnd.ekstep.html-archive" diff --git a/ansible/roles/lp-synctool-deploy/defaults/main.yml b/ansible/roles/lp-synctool-deploy/defaults/main.yml index b9919c28d4..ded17cf57a 100644 --- a/ansible/roles/lp-synctool-deploy/defaults/main.yml +++ b/ansible/roles/lp-synctool-deploy/defaults/main.yml @@ -23,6 +23,6 @@ redis_port: 6379 graphevent_kafka_url: "{{groups['processing-cluster-zookeepers']|join(':9092,')}}:9092" search_lms_index_host: "{{ groups['core-es']|join(':9200,')}}:9200" -cloud_store: azure +cloud_store: aws azure_public_container: azure_account_name: \ No newline at end of file diff --git a/ansible/roles/lp-synctool-deploy/templates/application.conf.j2 b/ansible/roles/lp-synctool-deploy/templates/application.conf.j2 index 686acdda15..03e546831a 100644 --- a/ansible/roles/lp-synctool-deploy/templates/application.conf.j2 +++ b/ansible/roles/lp-synctool-deploy/templates/application.conf.j2 @@ -81,6 +81,9 @@ cloud_storage_type="{{ cloud_store }}" azure_storage_key="{{sunbird_public_storage_account_name}}" azure_storage_secret="{{sunbird_public_storage_account_key}}" azure_storage_container="{{ azure_public_container }}" +aws_storage_key = "{{ aws_storage_key }}" +aws_storage_secret: "{{ aws_storage_secret }}" +aws_storage_container: "{{ aws_storage_container }}" contentTypeToPrimaryCategory { ClassroomTeachingVideo: "Explanation Content" diff --git a/ansible/roles/samza-jobs/tasks/deploy.yml b/ansible/roles/samza-jobs/tasks/deploy.yml index 2b9598eb02..feb0b54876 100644 --- a/ansible/roles/samza-jobs/tasks/deploy.yml +++ b/ansible/roles/samza-jobs/tasks/deploy.yml @@ -71,7 +71,17 @@ replace: dest="{{item[1].stdout}}" regexp="{{item[0].key}}" replace="{{item[0].value}}" when: "{{item[1]|changed}}" with_nested: - - [{key: "__yarn_host__", value: "{{__yarn_host__}}"}, {key: "__yarn_port__", value: "{{__yarn_port__}}"}, {key: "__env__", value: "{{env}}" }, {key: "__env_name__", value: "{{env_name}}" }, {key: "__zookeepers__", value: "{{zookeepers}}"}, {key: "__kafka_brokers__", value: "{{kafka_brokers}}"}, {key: "__delayInMilliSeconds__", value: "{{delayInMilliSeconds}}" }, {key: "__retryTimeInMilliSeconds__", value: "{{retryTimeInMilliSeconds}}" }, {key: "__bypass_reverse_search__", value: "{{bypass_reverse_search}}" }, {key: "__retryBackoffBaseInSeconds__", value: "{{retry_backoff_base_in_seconds}}" }, {key: "__retryLimit__", value: "{{retry_limit}}" }, {key: "__retryLimitEnable__", value: "{{retry_limit_enable}}" }, {key: "__google_api_key__", value: "{{google_api_key}}" }, {key: "__searchServiceEndpoint__", value: "{{search_service_endpoint}}" }, {key: "__objectDenormalizationAdditionalConfig__", value: "{{object_denormalization_additional_config}}" },{key: "__audit_es_host__", value: "{{audit_es_host}}"}, {key: "__search_es_host__", value: "{{search_es_host}}"}, {key: "__redis_host__", value: "{{redis_host}}"}, {key: "__dp_redis_host__", value: "{{dp_redis_host}}"}, {key: "__redis_port__", value: "{{redis_port}}"}, {key: "__environment_id__", value: "{{environment_id}}"}, {key: "__graph_passport_key__", value: "{{graph_passport_key}}"}, {key: "__lp_bolt_url__", value: "{{lp_bolt_url}}"}, {key: "__lp_bolt_read_url__", value: "{{lp_bolt_read_url}}"}, {key: "__lp_bolt_write_url__", value: "{{lp_bolt_write_url}}"}, {key: "__other_bolt_url__", value: "{{other_bolt_url}}"}, {key: "__other_bolt_read_url__", value: "{{other_bolt_read_url}}"}, {key: "__other_bolt_write_url__", value: "{{other_bolt_write_url}}"}, {key: "__mw_shard_id__", value: "{{mw_shard_id}}"}, {key: "__lp_url__", value: "{{lp_url}}"}, {key: "__cloud_storage_config_environment__", value: "{{cloud_storage_config_environment}}"}, {key: "__google_vision_tagging__", value: "{{google_vision_tagging}}"}, {key: "__lp_tmpfile_location__", value: "{{lp_tmpfile_location}}"}, {key: "__esRouterAdditionalConfig__", value: "{{es_router_additional_config}}"},{key: "__esRouterSecondaryAdditionalConfig__", value: "{{es_router_additional_secondary_config}}"},{key: "__es_port__", value: "{{es_port}}"}, {key: "__keyspace_name__", value: "{{content_keyspace_name}}"}, {key: "__collection_fullecar_disable__", value: "{{collection_fullecar_disable}}"},{key: "__max_iteration_count_for_samza_job__", value: "{{max_iteration_count_for_samza_job}}"},{key: "__cloud_storage_type__", value: "{{cloud_store}}"},{key: "__azure_storage_key__", value: "{{sunbird_public_storage_account_name}}"},{key: "__azure_storage_secret__", value: "{{sunbird_public_storage_account_key}}"},{key: "__azure_storage_container__", value: "{{azure_public_container}}"},{key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__plugin_media_base_url__", value: "{{plugin_media_base_url}}"}, {key: "__installation_id__", value: "{{instance_name}}"}, {key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__hierarchy_keyspace_name__", value: "{{hierarchy_keyspace_name}}"}, {key: "__composite_search_indexer_container_count__", value: "{{composite_search_indexer_container_count}}"},{key: "__cassandra_lp_connection__", value: "{{lp_cassandra_connection}}"}, {key: "__cassandra_lpa_connection__", value: "{{dp_cassandra_connection}}"}, {key: "__streaming_mime_type__", value: "{{streaming_mime_type}}"}, {key: "__cassandra_sunbird_connection__", value: "{{core_cassandra_connection}}"}, {key: "__cloud_upload_retry_count__", value: "{{cloud_upload_retry_count}}"}, {key: "__compositesearch_index_name__", value: "{{compositesearch_index_name}}"},{key: "__publish_pipeline_container_count__", value: "{{publish_pipeline_container_count}}"},{key: "__yarn_container_memory_mb__", value: "{{publish_yarn_container_memory_mb}}"},{key: "__youtube_api_key__", value: "{{youtube_api_key}}"},{key: "__kp_learning_service_base_url__", value: "{{kp_learning_service_base_url}}"},{key: "__sunbird_installation__", value: "{{sunbird_platform_installation}}"}, {key: "__search_lms_es_host__", value: "{{search_lms_es_host}}"},{key: "__dial_image_storage_container__", value: "{{dial_image_storage_container}}"},{key: "__dial_base_url__", value: "{{dial_base_url}}"},{key: "__learner_service_base_url__", value: "{{learner_service_base_url}}"},{key: "__cert_service_base_url__", value: "{{cert_service_base_url}}"},{key: "__certificate_base_path__", value: "{{certificate_base_path}}"},{key: "__kp_content_service_base_url__", value: "{{kp_content_service_base_url}}"},{key: "__kp_print_service_base_url__", value: "{{kp_print_service_base_url}}"},{key: "__cert_reg_service_base_url__", value: "{{cert_reg_service_base_url}}"},{key: "__kp_search_service_base_url__", value: "{{kp_search_service_base_url}}"},{key: "__samza_coordinator_replication_factor__", value: "{{samza_coordinator_replication_factor}}"},{key: "__samza_checkpoint_replication_factor__", value: "{{samza_checkpoint_replication_factor}}"},{key: "__course_batch_updater_container_count__", value: "{{course_batch_updater_container_count}}"},{key: "__course_certificate_generator_container_count__", value: "{{course_certificate_generator_container_count}}"},{key: "__course_progress_batch_size__", value: "{{course_progress_batch_size}}"},{key: "__itemset_generate_pdf__", value: "{{itemset_generate_pdf}}"},{key: "__auto_creator_container_count__", value: "{{auto_creator_container_count}}"},{key: "__content_streaming_enabled__", value: "{{content_streaming_enabled}}"},{key: "__lms_service_base_url__", value: "{{lms_service_base_url}}"},{key: "__mvc_search_indexer_container_count__", value: "{{mvc_search_indexer_container_count}}"}, {key: "__search_es7_host__", value: "{{search_es7_host}}"} , {key: "__ml-keywordapi__", value: "{{mlworkbench}}"},{key: "__auto_creator_artifact_allowed_sources__", value: "{{auto_creator_artifact_allowed_sources}}"},{key: "__publish_pipeline_task_opts__", value: "{{publish_pipeline_task_opts}}"},{key: "__auto_creator_g_service_acct_cred__", value: "{{auto_creator_gservice_acct_cred}}"},{key: "__certificate_pre_processor_container_count__", value: "{{certificate_pre_processor_container_count}}"},{key: "__master_category_validation_enabled__", value: "{{master_category_validation_enabled}}"}] + - [{key: "__yarn_host__", value: "{{__yarn_host__}}"}, {key: "__yarn_port__", value: "{{__yarn_port__}}"}, {key: "__env__", value: "{{env}}" }, {key: "__env_name__", value: "{{env_name}}" }, {key: "__zookeepers__", value: "{{zookeepers}}"}, {key: "__kafka_brokers__", value: "{{kafka_brokers}}"}, {key: "__delayInMilliSeconds__", value: "{{delayInMilliSeconds}}" }, {key: "__retryTimeInMilliSeconds__", value: "{{retryTimeInMilliSeconds}}" }, {key: "__bypass_reverse_search__", value: "{{bypass_reverse_search}}" }, {key: "__retryBackoffBaseInSeconds__", value: "{{retry_backoff_base_in_seconds}}" }, {key: "__retryLimit__", value: "{{retry_limit}}" }, {key: "__retryLimitEnable__", value: "{{retry_limit_enable}}" }, {key: "__google_api_key__", value: "{{google_api_key}}" }, {key: "__searchServiceEndpoint__", value: "{{search_service_endpoint}}" }, {key: "__objectDenormalizationAdditionalConfig__", value: "{{object_denormalization_additional_config}}" },{key: "__audit_es_host__", value: "{{audit_es_host}}"}, {key: "__search_es_host__", value: "{{search_es_host}}"}, {key: "__redis_host__", value: "{{redis_host}}"}, {key: "__dp_redis_host__", value: "{{dp_redis_host}}"}, {key: "__redis_port__", value: "{{redis_port}}"}, {key: "__environment_id__", value: "{{environment_id}}"}, {key: "__graph_passport_key__", value: "{{graph_passport_key}}"}, {key: "__lp_bolt_url__", value: "{{lp_bolt_url}}"}, {key: "__lp_bolt_read_url__", value: "{{lp_bolt_read_url}}"}, {key: "__lp_bolt_write_url__", value: "{{lp_bolt_write_url}}"}, {key: "__other_bolt_url__", value: "{{other_bolt_url}}"}, {key: "__other_bolt_read_url__", value: "{{other_bolt_read_url}}"}, {key: "__other_bolt_write_url__", value: "{{other_bolt_write_url}}"}, {key: "__mw_shard_id__", value: "{{mw_shard_id}}"}, {key: "__lp_url__", value: "{{lp_url}}"}, {key: "__cloud_storage_config_environment__", value: "{{cloud_storage_config_environment}}"}, {key: "__google_vision_tagging__", value: "{{google_vision_tagging}}"}, {key: "__lp_tmpfile_location__", value: "{{lp_tmpfile_location}}"}, {key: "__esRouterAdditionalConfig__", value: "{{es_router_additional_config}}"},{key: "__esRouterSecondaryAdditionalConfig__", value: "{{es_router_additional_secondary_config}}"},{key: "__es_port__", value: "{{es_port}}"}, {key: "__keyspace_name__", value: "{{content_keyspace_name}}"}, {key: "__collection_fullecar_disable__", value: "{{collection_fullecar_disable}}"},{key: "__max_iteration_count_for_samza_job__", value: "{{max_iteration_count_for_samza_job}}"},{key: "__cloud_storage_type__", value: "{{cloud_store}}"},{key: "__aws_access_key_id__",value: "{{aws_storage_key}}"},{key: "__aws_secret_access_key__",value: "{{aws_storage_secret}}"},{key: "__aws_storage_container__",value: "{{aws_storage_container}}"},{key: "__azure_storage_key__", value: "{{sunbird_public_storage_account_name}}"},{key: "__azure_storage_secret__", value: "{{sunbird_public_storage_account_key}}"},{key: "__azure_storage_container__", value: "{{azure_public_container}}"},{key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__plugin_media_base_url__", value: "{{plugin_media_base_url}}"}, {key: "__installation_id__", value: "{{instance_name}}"}, {key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__hierarchy_keyspace_name__", value: "{{hierarchy_keyspace_name}}"}, {key: "__composite_search_indexer_container_count__", value: "{{composite_search_indexer_container_count}}"},{key: "__cassandra_lp_connection__", value: "{{lp_cassandra_connection}}"}, {key: "__cassandra_lpa_connection__", value: "{{dp_cassandra_connection}}"}, {key: "__streaming_mime_type__", value: "{{streaming_mime_type}}"}, {key: "__cassandra_sunbird_connection__", value: "{{core_cassandra_connection}}"}, {key: "__cloud_upload_retry_count__", value: "{{cloud_upload_retry_count}}"}, {key: "__compositesearch_index_name__", value: "{{compositesearch_index_name}}"},{key: "__publish_pipeline_container_count__", value: "{{publish_pipeline_container_count}}"},{key: "__yarn_container_memory_mb__", value: "{{publish_yarn_container_memory_mb}}"},{key: "__youtube_api_key__", value: "{{youtube_api_key}}"},{key: "__kp_learning_service_base_url__", value: "{{kp_learning_service_base_url}}"},{key: "__sunbird_installation__", value: "{{sunbird_platform_installation}}"}, {key: "__search_lms_es_host__", value: "{{search_lms_es_host}}"},{key: "__dial_image_storage_container__", value: "{{dial_image_storage_container}}"},{key: "__dial_base_url__", value: "{{dial_base_url}}"},{key: "__learner_service_base_url__", value: "{{learner_service_base_url}}"},{key: "__cert_service_base_url__", value: "{{cert_service_base_url}}"},{key: "__certificate_base_path__", value: "{{certificate_base_path}}"},{key: "__kp_content_service_base_url__", value: "{{kp_content_service_base_url}}"},{key: "__kp_print_service_base_url__", value: "{{kp_print_service_base_url}}"},{key: "__cert_reg_service_base_url__", value: "{{cert_reg_service_base_url}}"},{key: "__kp_search_service_base_url__", value: "{{kp_search_service_base_url}}"},{key: "__samza_coordinator_replication_factor__", value: "{{samza_coordinator_replication_factor}}"},{key: "__samza_checkpoint_replication_factor__", value: "{{samza_checkpoint_replication_factor}}"},{key: "__course_batch_updater_container_count__", value: "{{course_batch_updater_container_count}}"},{key: "__course_certificate_generator_container_count__", value: "{{course_certificate_generator_container_count}}"},{key: "__course_progress_batch_size__", value: "{{course_progress_batch_size}}"},{key: "__itemset_generate_pdf__", value: "{{itemset_generate_pdf}}"},{key: "__auto_creator_container_count__", value: "{{auto_creator_container_count}}"},{key: "__content_streaming_enabled__", value: "{{content_streaming_enabled}}"},{key: "__lms_service_base_url__", value: "{{lms_service_base_url}}"},{key: "__mvc_search_indexer_container_count__", value: "{{mvc_search_indexer_container_count}}"}, {key: "__search_es7_host__", value: "{{search_es7_host}}"} , {key: "__ml-keywordapi__", value: "{{mlworkbench}}"},{key: "__auto_creator_artifact_allowed_sources__", value: "{{auto_creator_artifact_allowed_sources}}"},{key: "__publish_pipeline_task_opts__", value: "{{publish_pipeline_task_opts}}"},{key: "__auto_creator_g_service_acct_cred__", value: "{{auto_creator_gservice_acct_cred}}"}] + - "{{ (config_files|default({})).results|default([]) }}" + + +@@ -101,4 +101,4 @@ + args: + chdir: "{{samza_jobs_dir}}/extract/" + +- file: path={{samza_jobs_dir}} owner=hduser group=hadoop state=directory recurse=yes +- file: path={{samza_jobs_dir}} owner=hduser group=hadoop state=directory recurse=yes + - "{{ (config_files|default({})).results|default([]) }}" diff --git a/docs/domain_model_v2/definitions/content_definition.json b/docs/domain_model_v2/definitions/content_definition.json index 8e0a97a6c8..c831063254 100644 --- a/docs/domain_model_v2/definitions/content_definition.json +++ b/docs/domain_model_v2/definitions/content_definition.json @@ -271,7 +271,9 @@ "audio/ogg", "audio/webm", "audio/x-wav", - "audio/wav" + "audio/wav", + "application/json", + "application/quiz" ], "defaultValue": "application/vnd.ekstep.ecml-archive", "renderingHints": "{ 'inputType': 'select', 'order': 9 }", diff --git a/docs/domain_model_v2/definitions/content_image_definition.json b/docs/domain_model_v2/definitions/content_image_definition.json index d4d6b09c65..4ab316951f 100644 --- a/docs/domain_model_v2/definitions/content_image_definition.json +++ b/docs/domain_model_v2/definitions/content_image_definition.json @@ -271,7 +271,9 @@ "audio/ogg", "audio/webm", "audio/x-wav", - "audio/wav" + "audio/wav", + "application/json", + "application/quiz" ], "defaultValue": "application/vnd.ekstep.ecml-archive", "renderingHints": "{ 'inputType': 'select', 'order': 9 }", diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 03cdce2f6b..961a7b3cc4 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -39,17 +39,17 @@ producer_max_request_size: 1572864 ### Activity Aggregate job related vars activity_agg_consumer_parallelism: 1 -activity_agg_dedup_parallelism: 1 -activity_agg_parallelism: 1 +activity_agg_dedup_parallelism: "{{ activity_aggregate_updater_replica | default('1') }}" +activity_agg_parallelism: "{{ activity_aggregate_updater_replica | default('1') }}" enrolment_complete_parallelism: 1 middleware_consumption_table: "user_content_consumption" middleware_user_activity_agg_table: "user_activity_agg" -activity_agg_checkpointing_interval: 300000 -activity_agg_checkpointing_pause_interval: 90000 +activity_agg_checkpointing_interval: 180000 +activity_agg_checkpointing_pause_interval: 60000 activity_agg_batch_interval: 60 activity_agg_batch_read_size: 1 activity_agg_batch_write_size: 10 -activity_agg_window_shards: 1000 +activity_agg_window_shards: 30 activity_agg_dedup_index: 13 activity_agg_dedup_expiry: 604800 activity_module_aggs_enabled: true @@ -140,7 +140,7 @@ flink_job_names: cpu_requests: 0.3 activity-aggregate-updater: job_class_name: 'org.sunbird.job.aggregate.task.ActivityAggregateUpdaterStreamTask' - replica: 1 + replica: "{{ activity_aggregate_updater_replica | default('1') }}" jobmanager_memory: 1024m taskmanager_memory: 1024m taskslots: 1 diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index f13eea269b..0a7b64ee36 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -86,6 +86,15 @@ spec: prometheus.io/scrape: 'true' prometheus.io/port: "{{ .Values.jobmanager.prom_port }}" spec: + {{- if .Values.tolerations.enabled }} + tolerations: + - key: "dedicated" + operator: "Equal" + value: "flink-prod" + effect: "NoSchedule" + nodeSelector: + role: flink + {{- end }} volumes: - name: flink-config-volume configMap: @@ -156,18 +165,30 @@ kind: Deployment metadata: name: {{ .Release.Name }}-taskmanager namespace: {{ .Values.namespace }} + spec: replicas: {{ .Values.taskmanager.replicas }} selector: matchLabels: app: flink component: {{ .Release.Name }}-taskmanager + template: metadata: labels: app: flink component: {{ .Release.Name }}-taskmanager + spec: + {{- if .Values.tolerations.enabled }} + tolerations: + - key: "dedicated" + operator: "Equal" + value: "flink-prod" + effect: "NoSchedule" + nodeSelector: + role: flink + {{- end }} volumes: - name: flink-config-volume configMap: diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 32b744c1df..60f76618db 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -120,6 +120,7 @@ base_config: | } schema { basePath = "{{ kp_schema_base_path }}" + base_path= "{{ kp_schema_base_path }}" supportedVersion = { itemset = "2.0" } @@ -133,7 +134,7 @@ activity-aggregate-updater: output.audit.topic = {{ env_name }}.telemetry.raw output.failed.topic = {{ env_name }}.activity.agg.failed output.certissue.topic = {{ env_name }}.issue.certificate.request - groupId = {{ env_name }}-activity-aggregate-group + groupId = {{ env_name }}-activity-aggregate-group2 } task { window.shards = {{ activity_agg_window_shards }} @@ -291,16 +292,20 @@ questionset-publish: aws_storage_key = "{{ aws_storage_key }}" aws_storage_secret: "{{ aws_storage_secret }}" aws_storage_container: "{{ aws_storage_container }}" + master.category.validation.enabled ="{{ master_category_validation_enabled }}" flink-conf: |+ jobmanager.memory.flink.size: {{ flink_job_names['questionset-publish'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['questionset-publish'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['questionset-publish'].taskslots }} - parallelism.default: 1 env.java.opts: -Dlog4j2.formatMsgNoLookups=true + parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 + metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter + metrics.reporter.prom.port: 9250-9260 + video-stream-generator: video-stream-generator: |+ @@ -376,7 +381,7 @@ search-indexer: dialcode.index.name = "dialcode" dailcodemetrics.index.name = "dialcodemetrics" restrict.metadata.objectTypes = [] - nested.fields = ["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials", "provider", "osMetadata", "actions", "transcripts", "accessibility","taxonomyPaths_v2", "competencies_v3"] + nested.fields = ["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials", "provider", "osMetadata", "actions", "transcripts", "accessibility", "taxonomyPaths_v2", "competencies_v3"] schema.definition_cache.expiry = {{ schema_definition_cache_expiry_in_sec }} restrict.objectTypes = ["EventSet", "EventSetImage", "Event", "EventImage", "Questionnaire", "Misconception", "FrameworkType", "Concept", "Misconception", "Language", "Reference", "Dimension", "Method", "Library", "Domain", "Api"] ignored.fields={{ search_indexer_ignored_fields | to_json }} @@ -432,6 +437,7 @@ enrolment-reconciliation: jobmanager.memory.flink.size: {{ flink_job_names['enrolment-reconciliation'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['enrolment-reconciliation'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['enrolment-reconciliation'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 @@ -711,6 +717,7 @@ collection-cert-pre-processor: keyspace = "sunbird" table = "user_assessment_summary" } + cert_domain_url = "{{ cert_domain_url }}" user_read_api = "/private/user/v1/read" content_read_api = "/content/v3/read" @@ -765,6 +772,8 @@ collection-certificate-generator: cert_cloud_storage_type = "{{ cert_cloud_storage_type }}" cert_aws_storage_secret = "{{ cert_aws_storage_secret }}" cert_aws_storage_key = "{{ cert_aws_storage_key }}" + + service { certreg.basePath = "{{ cert_reg_service_base_url }}" learner.basePath = "{{ learner_service_base_url }}" @@ -867,7 +876,7 @@ certificate-email-service: parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 - + content-publish: content-publish: |+ include file("/data/flink/conf/base-config.conf") @@ -964,10 +973,8 @@ content-publish: spine = ["appIcon"] full = ["appIcon", "grayScaleAppIcon", "artifactUrl", "itemSetPreviewUrl", "media"] } - nested.fields=["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials", "provider", "osMetadata", "actions", "transcripts", "accessibility","taxonomyPaths_v2", "competencies_v3", "test_competencies_v2","competencies_v2"] - restrict.objectTypes = ["EventSet", "Questionnaire", "Misconception", "FrameworkType", "EventSetImage", "EventImage"] - + restrict.objectTypes = ["EventSet", "Questionnaire", "Misconception", "FrameworkType", "EventSetImage", "EventImage"] } cloud_storage { folder { @@ -980,7 +987,6 @@ content-publish: keyspace = "{{ hierarchy_keyspace_name }}" table = "content_hierarchy" } - contentTypeToPrimaryCategory { ClassroomTeachingVideo: "Explanation Content" ConceptMap: "Learning Resource" @@ -1017,41 +1023,6 @@ content-publish: CourseUnit: "Course Unit" TextBookUnit: "Textbook Unit" Asset: "Certificate Template" - } - contentTypeToPrimaryCategory { - ClassroomTeachingVideo: "Explanation Content" - ConceptMap: "Learning Resource" - Course: "Course" - CuriosityQuestionSet: "Practice Question Set" - eTextBook: "eTextbook" - ExperientialResource: "Learning Resource" - ExplanationResource: "Explanation Content" - ExplanationVideo: "Explanation Content" - FocusSpot: "Teacher Resource" - LearningOutcomeDefinition: "Teacher Resource" - MarkingSchemeRubric: "Teacher Resource" - PedagogyFlow: "Teacher Resource" - PracticeQuestionSet: "Practice Question Set" - PracticeResource: "Practice Question Set" - SelfAssess: "Course Assessment" - TeachingMethod: "Teacher Resource" - TextBook: "Digital Textbook" - Collection: "Content Playlist" - ExplanationReadingMaterial: "Learning Resource" - LearningActivity: "Learning Resource" - LessonPlan: "Content Playlist" - LessonPlanResource: "Teacher Resource" - PreviousBoardExamPapers: "Learning Resource" - TVLesson: "Explanation Content" - OnboardingResource: "Learning Resource" - ReadingMaterial: "Learning Resource" - Template: "Template" - Asset: "Asset" - Plugin: "Plugin" - LessonPlanUnit: "Lesson Plan Unit" - CourseUnit: "Course Unit" - TextBookUnit: "Textbook Unit" - Asset: "Certificate Template" Course: "Program" } compositesearch.index.name = "compositesearch" @@ -1064,11 +1035,10 @@ content-publish: aws_storage_key="{{ aws_storage_key }}" aws_storage_secret="{{ aws_storage_secret }}" aws_storage_container="{{ aws_storage_container }}" - + Course="Program" Course="Mandatory Course Goal" - master.category.validation.enabled ="{{ master_category_validation_enabled }}" service { print.basePath = "{{ kp_print_service_base_url }}" @@ -1126,3 +1096,6 @@ qrcode-image-generator: parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 +tolerations: + enabled: {{ kp_flink_tolerations_enabled | default('true') }} + diff --git a/platform-jobs/samza/auto-creator/src/main/config/auto-creator.properties b/platform-jobs/samza/auto-creator/src/main/config/auto-creator.properties index b3f049495b..e5666efbe1 100644 --- a/platform-jobs/samza/auto-creator/src/main/config/auto-creator.properties +++ b/platform-jobs/samza/auto-creator/src/main/config/auto-creator.properties @@ -85,5 +85,9 @@ aws_storage_key=__aws_access_key_id__ aws_storage_secret=__aws_secret_access_key__ aws_storage_container=__aws_storage_container__ +cephs3_storage_key=__cephs3_access_key_id__ +cephs3_storage_secret=__cephs3_secret_access_key__ +cephs3_storage_container=__cephs3_storage_container__ +cephs3_storage_endpoint=__cephs3_storage_endpoint__ diff --git a/platform-jobs/samza/auto-creator/src/main/config/local.auto-creator.properties.properties b/platform-jobs/samza/auto-creator/src/main/config/local.auto-creator.properties.properties index 45783a38b0..9491fc6a2c 100644 --- a/platform-jobs/samza/auto-creator/src/main/config/local.auto-creator.properties.properties +++ b/platform-jobs/samza/auto-creator/src/main/config/local.auto-creator.properties.properties @@ -72,3 +72,9 @@ aws_storage_key=__aws_access_key_id__ aws_storage_secret=__aws_secret_access_key__ aws_storage_container=__aws_storage_container__ + +cephs3_storage_key=__cephs3_access_key_id__ +cephs3_storage_secret=__cephs3_secret_access_key__ +cephs3_storage_container=__cephs3_storage_container__ +cephs3_storage_endpoint=__cephs3_storage_endpoint__ + diff --git a/platform-jobs/samza/publish-pipeline/src/main/config/publish-pipeline.properties b/platform-jobs/samza/publish-pipeline/src/main/config/publish-pipeline.properties index fc8c47972b..046e2e8a22 100644 --- a/platform-jobs/samza/publish-pipeline/src/main/config/publish-pipeline.properties +++ b/platform-jobs/samza/publish-pipeline/src/main/config/publish-pipeline.properties @@ -86,6 +86,11 @@ cloud_storage.media.folder=media cloud_storage.ecar.folder=ecar_files cloud_storage.upload.url.ttl=600 +cephs3_storage_key=__cephs3_access_key_id__ +cephs3_storage_secret=__cephs3_secret_access_key__ +cephs3_storage_container=__cephs3_storage_container__ +cephs3_storage_endpoint=__cephs3_storage_endpoint__ + # Media download configuration content.media.base.url=__content_media_base_url__ diff --git a/platform-jobs/samza/qrcode-image-generator/src/main/config/local.qrcode-image-generator.properties b/platform-jobs/samza/qrcode-image-generator/src/main/config/local.qrcode-image-generator.properties index 55f84fae02..f025402c82 100644 --- a/platform-jobs/samza/qrcode-image-generator/src/main/config/local.qrcode-image-generator.properties +++ b/platform-jobs/samza/qrcode-image-generator/src/main/config/local.qrcode-image-generator.properties @@ -55,6 +55,11 @@ aws_storage_secret=__aws_secret_access_key__ aws_storage_container=__aws_storage_container__ cloud_upload_retry_count=3 +cephs3_storage_key=__cephs3_access_key_id__ +cephs3_storage_secret=__cephs3_secret_access_key__ +cephs3_storage_container=__cephs3_storage_container__ +cephs3_storage_endpoint=__cephs3_storage_endpoint__ + # Cassandra connection details cassandra.lp.connection=localhost:9042 cassandra.lpa.connection=localhost:9042 diff --git a/platform-jobs/samza/qrcode-image-generator/src/main/config/qrcode-image-generator.properties b/platform-jobs/samza/qrcode-image-generator/src/main/config/qrcode-image-generator.properties index b5954fd766..b8b030df33 100644 --- a/platform-jobs/samza/qrcode-image-generator/src/main/config/qrcode-image-generator.properties +++ b/platform-jobs/samza/qrcode-image-generator/src/main/config/qrcode-image-generator.properties @@ -55,6 +55,11 @@ aws_storage_secret=__aws_secret_access_key__ aws_storage_container=__aws_storage_container__ cloud_upload_retry_count=__cloud_upload_retry_count__ +cephs3_storage_key=__cephs3_access_key_id__ +cephs3_storage_secret=__cephs3_secret_access_key__ +cephs3_storage_container=__cephs3_storage_container__ +cephs3_storage_endpoint=__cephs3_storage_endpoint__ + # Cassandra connection details cassandra.lp.connection=__cassandra_lp_connection__ cassandra.lpa.connection=__cassandra_lpa_connection__ diff --git a/platform-jobs/samza/qrcode-image-generator/src/main/java/org/sunbird/jobs/samza/util/CloudStorageUtil.java b/platform-jobs/samza/qrcode-image-generator/src/main/java/org/sunbird/jobs/samza/util/CloudStorageUtil.java index 81493cae58..35abb3967f 100644 --- a/platform-jobs/samza/qrcode-image-generator/src/main/java/org/sunbird/jobs/samza/util/CloudStorageUtil.java +++ b/platform-jobs/samza/qrcode-image-generator/src/main/java/org/sunbird/jobs/samza/util/CloudStorageUtil.java @@ -26,11 +26,16 @@ public class CloudStorageUtil { if(StringUtils.equalsIgnoreCase(cloudStoreType, "azure")) { String storageKey = Platform.config.getString("azure_storage_key"); String storageSecret = Platform.config.getString("azure_storage_secret"); - storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret)); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.empty())); }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "aws")) { String storageKey = Platform.config.getString("aws_storage_key"); String storageSecret = Platform.config.getString("aws_storage_secret"); - storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret)); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.empty())); + }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "cephs3")) { + String storageKey = Platform.config.getString("cephs3_storage_key"); + String storageSecret = Platform.config.getString("cephs3_storage_secret"); + String endPoint = Platform.config.getString("cephs3_storage_endpoint"); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.apply(endPoint))); }else { throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Error while initialising cloud storage"); } diff --git a/platform-modules/actors/src/main/java/org/sunbird/learning/util/CloudStore.java b/platform-modules/actors/src/main/java/org/sunbird/learning/util/CloudStore.java index 32643d61e8..942df9881a 100644 --- a/platform-modules/actors/src/main/java/org/sunbird/learning/util/CloudStore.java +++ b/platform-modules/actors/src/main/java/org/sunbird/learning/util/CloudStore.java @@ -28,11 +28,16 @@ public class CloudStore { if(StringUtils.equalsIgnoreCase(cloudStoreType, "azure")) { String storageKey = Platform.config.getString("azure_storage_key"); String storageSecret = Platform.config.getString("azure_storage_secret"); - storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret)); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.empty())); }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "aws")) { String storageKey = Platform.config.getString("aws_storage_key"); String storageSecret = Platform.config.getString("aws_storage_secret"); - storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret)); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.empty())); + }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "cephs3")) { + String storageKey = Platform.config.getString("cephs3_storage_key"); + String storageSecret = Platform.config.getString("cephs3_storage_secret"); + String endPoint = Platform.config.getString("cephs3_storage_endpoint"); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.apply(endPoint))); }else { throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Error while initialising cloud storage"); } @@ -47,6 +52,8 @@ public static String getContainerName() { return Platform.config.getString("azure_storage_container"); }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "aws")) { return S3PropertyReader.getProperty("aws_storage_container"); + }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "cephs3")) { + return S3PropertyReader.getProperty("cephs3_storage_container"); }else { throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Error while getting container name"); } diff --git a/platform-modules/common/src/main/java/org/sunbird/common/util/HttpDownloadUtility.java b/platform-modules/common/src/main/java/org/sunbird/common/util/HttpDownloadUtility.java index 6939f89057..1c75e1d204 100644 --- a/platform-modules/common/src/main/java/org/sunbird/common/util/HttpDownloadUtility.java +++ b/platform-modules/common/src/main/java/org/sunbird/common/util/HttpDownloadUtility.java @@ -11,6 +11,7 @@ import java.net.URL; import java.util.List; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.sunbird.common.Slug; import org.sunbird.telemetry.logger.TelemetryManager; @@ -24,7 +25,7 @@ public class HttpDownloadUtility { private static final int BUFFER_SIZE = 4096; - + /** * Downloads a file from a URL * @@ -34,6 +35,26 @@ public class HttpDownloadUtility { * path of the directory to save the file */ public static File downloadFile(String fileURL, String saveDir) { + try { + System.out.println("http utility called........."); + return download(fileURL, false); + } catch (Exception e) { + e.printStackTrace(); + TelemetryManager.error("Error! While Downloading File:"+ e.getMessage(), e); + } + TelemetryManager.warn("Something Went Wrong While Downloading the File '" + fileURL + "' returning 'null'. File url: "+ fileURL); + return null; + } + + /** + * Downloads a file from a URL + * + * @param fileURL + * HTTP URL of the file to be downloaded + * @param saveDir + * path of the directory to save the file + */ + public static File downloadFileOld(String fileURL, String saveDir) { HttpURLConnection httpConn = null; InputStream inputStream = null; FileOutputStream outputStream = null; @@ -158,5 +179,18 @@ public static String readFromUrl(String url) { TelemetryManager.log("Data read from url: " + sb.toString()); return sb.toString(); } + + public static File download(String artifactUrl, boolean extractFile) throws Exception { + if(StringUtils.isNotBlank(artifactUrl)){ + String localPath = "tmp/" + artifactUrl.trim() + File.separator; + String[] fileUrl = artifactUrl.split("/"); + String filename = fileUrl[fileUrl.length - 1]; + + File file = new File(localPath + filename); + FileUtils.copyURLToFile(new URL(artifactUrl), file); + return file; + } + return null; + } } \ No newline at end of file diff --git a/platform-modules/content-manager/src/main/java/org/sunbird/content/mimetype/mgr/impl/HTMLMimeTypeMgrImpl.java b/platform-modules/content-manager/src/main/java/org/sunbird/content/mimetype/mgr/impl/HTMLMimeTypeMgrImpl.java index f63ad30e7e..188e57bdae 100644 --- a/platform-modules/content-manager/src/main/java/org/sunbird/content/mimetype/mgr/impl/HTMLMimeTypeMgrImpl.java +++ b/platform-modules/content-manager/src/main/java/org/sunbird/content/mimetype/mgr/impl/HTMLMimeTypeMgrImpl.java @@ -5,6 +5,7 @@ import java.util.Map; import org.apache.commons.lang3.BooleanUtils; +import org.sunbird.common.Platform; import org.sunbird.common.dto.Response; import org.sunbird.common.exception.ResponseCode; import org.sunbird.content.common.ContentOperations; @@ -42,7 +43,9 @@ public class HTMLMimeTypeMgrImpl extends BaseMimeTypeManager implements IMimeTyp @Override public Response upload(String contentId, Node node, File uploadFile, boolean isAsync) { TelemetryManager.log("Calling Upload Content For Node ID: " + node.getIdentifier() + " Uploaded File :" + uploadFile); - if (hasGivenFile(uploadFile, "index.html")) { + Boolean indexHtmlValidation = (Platform.config.hasPath("indexHtmlValidation.env"))? Platform.config.getBoolean("indexHtmlValidation.env") : false; + Boolean flag = indexHtmlValidation ? hasGivenFile(uploadFile, "index.html") : true; + if (flag) { return uploadContentArtifact(contentId, node, uploadFile, false); } else { return ERROR(ContentErrorCodes.ERR_CONTENT_UPLOAD_FILE.name(), "Zip file doesn't have required files.", ResponseCode.CLIENT_ERROR); diff --git a/platform-modules/content-manager/src/main/java/org/sunbird/content/operation/finalizer/ReviewFinalizer.java b/platform-modules/content-manager/src/main/java/org/sunbird/content/operation/finalizer/ReviewFinalizer.java index 21111b918f..c2635bb900 100644 --- a/platform-modules/content-manager/src/main/java/org/sunbird/content/operation/finalizer/ReviewFinalizer.java +++ b/platform-modules/content-manager/src/main/java/org/sunbird/content/operation/finalizer/ReviewFinalizer.java @@ -270,7 +270,9 @@ private void validateResource(String collectionId){ notPublishedList.add(resource.getIdentifier()); } } - if(CollectionUtils.isNotEmpty(notPublishedList)){ + Boolean childPublishEnabled = Platform.config.hasPath("collection.only_published_children") ? + Platform.config.getBoolean("collection.only_published_children") : true; + if(!childPublishEnabled && CollectionUtils.isNotEmpty(notPublishedList)){ TelemetryManager.error("Collection: " + collectionId + " has unpublished resources:: " + notPublishedList); throw new ClientException("ERR_COLLECTION_WITH_UNPUBLISHED_RESOURCE", "Collection: " + collectionId + " has unpublished resources:: " + notPublishedList); } diff --git a/platform-modules/pom.xml b/platform-modules/pom.xml index 94cdc3d8ee..ad3a7e306b 100644 --- a/platform-modules/pom.xml +++ b/platform-modules/pom.xml @@ -19,7 +19,7 @@ 2.3.1 1.8 1.8 - 1.2.8 + 1.3.0-beta diff --git a/platform-modules/sunbird-framework/src/main/java/org/sunbird/framework/mgr/impl/FrameworkManagerImpl.java b/platform-modules/sunbird-framework/src/main/java/org/sunbird/framework/mgr/impl/FrameworkManagerImpl.java index 96caddf7cb..90bbb64430 100644 --- a/platform-modules/sunbird-framework/src/main/java/org/sunbird/framework/mgr/impl/FrameworkManagerImpl.java +++ b/platform-modules/sunbird-framework/src/main/java/org/sunbird/framework/mgr/impl/FrameworkManagerImpl.java @@ -201,6 +201,7 @@ public Response publishFramework(String frameworkId, String channelId) throws Ex } if (StringUtils.isNotBlank(frameworkId) && validateObject(frameworkId)) { generateFrameworkHierarchy(frameworkId); + FrameworkCache.delete(frameworkId); Response response = OK(); response.put(FrameworkEnum.publishStatus.name(), "Publish Operation for Framework Id '" + frameworkId + "' Started Successfully!"); diff --git a/platform-tools/spikes/content-tool/pom.xml b/platform-tools/spikes/content-tool/pom.xml index a485ae2693..22f0ceff9c 100644 --- a/platform-tools/spikes/content-tool/pom.xml +++ b/platform-tools/spikes/content-tool/pom.xml @@ -66,7 +66,7 @@ org.sunbird cloud-store-sdk - 1.2.5 + 1.3.0-beta diff --git a/platform-tools/spikes/content-tool/src/main/java/org/sunbird/content/tool/CloudStoreManager.java b/platform-tools/spikes/content-tool/src/main/java/org/sunbird/content/tool/CloudStoreManager.java index d4f0e23e60..34ac6cb41e 100644 --- a/platform-tools/spikes/content-tool/src/main/java/org/sunbird/content/tool/CloudStoreManager.java +++ b/platform-tools/spikes/content-tool/src/main/java/org/sunbird/content/tool/CloudStoreManager.java @@ -22,9 +22,10 @@ public class CloudStoreManager { protected String destStorageType = Platform.config.getString("destination.storage_type"); - protected BaseStorageService awsService = StorageServiceFactory.getStorageService(new StorageConfig("aws", Platform.config.getString("aws_storage_key"), Platform.config.getString("aws_storage_secret"))); - protected BaseStorageService azureService = StorageServiceFactory.getStorageService((new StorageConfig("azure", Platform.config.getString("azure_storage_key"), Platform.config.getString("azure_storage_secret")))); - private String cloudSrcBaseURL = Platform.config.getString("cloud.src.baseurl"); + protected BaseStorageService awsService = StorageServiceFactory.getStorageService(new StorageConfig("aws", Platform.config.getString("aws_storage_key"), Platform.config.getString("aws_storage_secret"), Option.empty())); + protected BaseStorageService azureService = StorageServiceFactory.getStorageService((new StorageConfig("azure", Platform.config.getString("azure_storage_key"), Platform.config.getString("azure_storage_secret"), Option.empty()))); + protected BaseStorageService cephService = StorageServiceFactory.getStorageService((new StorageConfig("cephs3", Platform.config.getString("cephs3_storage_key"), Platform.config.getString("cephs3_storage_secret"), Option.apply(Platform.config.getString("cephs3_storage_endpoint"))))); + private String cloudSrcBaseURL = Platform.config.getString("cloud.src.baseurl"); private String cloudDestBaseURL = Platform.config.getString("cloud.dest.baseurl"); @@ -239,6 +240,8 @@ public String getContainerName(String cloudStoreType) { return Platform.config.getString("azure_storage_container"); }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "aws")) { return Platform.config.getString("aws_storage_container"); + }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "cephs3")) { + return Platform.config.getString("cephs3_storage_container"); }else { throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Error while getting container name"); } @@ -249,6 +252,8 @@ public BaseStorageService getcloudService(String cloudStoreType){ return azureService; }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "aws")) { return awsService; + }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "cephs3")) { + return cephService; }else { throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Error while getting container name"); }