Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions model/src/main/proto/topic.proto
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,9 @@ message ConceptCard {
// The core explanation of the skill being reviewed.
SubtitledHtml explanation = 3;

// A list of worked examples to present to the learner.
repeated SubtitledHtml worked_example = 4;
// This field used to be a list of worked examples to present to the learner.
// The field was never used, and was removed in Oct 2025.
reserved 4;

// Mapping from content_id to a VoiceoverMapping for each SubtitledHtml in this concept card that
// has corresponding recorded audio to play.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,9 @@ class LessonListDownloader(

fun downloadLessonListAsync(lessonListOutputFile: File): Deferred<Unit> {
return CoroutineScope(scriptBgDispatcher).async {
if (apiDebugDir != null) println("Config: Using ${apiDebugDir.path}/ for storing API responses (for debugging).")
if (apiDebugDir != null) {
println("Config: Using ${apiDebugDir.path}/ for storing API responses (for debugging).")
}

val listResponse = downloadTopicListResponseDto()
println()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,10 @@ import kotlinx.coroutines.flow.consumeAsFlow
import kotlinx.coroutines.flow.launchIn
import kotlinx.coroutines.flow.onEach
import kotlinx.coroutines.flow.withIndex
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withContext
import org.oppia.android.scripts.assets.DtoProtoToLegacyProtoConverter.convertToClassroomIdList
import org.oppia.android.scripts.assets.DtoProtoToLegacyProtoConverter.convertToClassroomRecord
import org.oppia.android.scripts.assets.DtoProtoToLegacyProtoConverter.convertToConceptCardList
import org.oppia.android.scripts.assets.DtoProtoToLegacyProtoConverter.convertToExploration
import org.oppia.android.scripts.assets.DtoProtoToLegacyProtoConverter.convertToStoryRecord
Expand Down Expand Up @@ -95,12 +96,10 @@ import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import org.oppia.android.scripts.assets.DtoProtoToLegacyProtoConverter.convertToClassroomIdList
import org.oppia.android.scripts.assets.DtoProtoToLegacyProtoConverter.convertToClassroomRecord
import kotlin.system.exitProcess
import org.oppia.proto.v1.api.DownloadRequestStructureIdentifierDto.Builder as DownloadReqStructIdDtoBuilder
import org.oppia.proto.v1.structure.DragAndDropSortInputInstanceDto.RuleSpecDto as DragDropSortRuleSpecDto
import org.oppia.proto.v1.structure.ItemSelectionInputInstanceDto.RuleSpecDto as ItemSelRuleSpecDto
import kotlin.system.exitProcess

/*
TODO: Thoughts for improving the script:
Expand Down Expand Up @@ -236,7 +235,9 @@ class LessonDownloader(
private val memoizedLoadedImageData by lazy { ConcurrentHashMap<File, ByteArray>() }

fun downloadLessons(outputDir: File, failOnError: Boolean) {
val downloadDeferred = CoroutineScope(coroutineDispatcher).async { downloadAllLessons(outputDir, failOnError) }
val downloadDeferred = CoroutineScope(coroutineDispatcher).async {
downloadAllLessons(outputDir, failOnError)
}
val exceptionResult = runBlocking {
runCatching { downloadDeferred.await() }.also { shutdownBlocking() }
}
Expand Down Expand Up @@ -711,7 +712,10 @@ class LessonDownloader(
issues.forEach { issue ->
val missingLangs = issue.missingLanguages.joinToString { it.name }
val presentLangs = issue.presentLanguages.joinToString { it.name }
println(" - Image ${issue.filename} exists in languages: $presentLangs, but is missing in: $missingLangs")
println(
" - Image ${issue.filename} exists in languages: $presentLangs, " +
"but is missing in: $missingLangs"
)
}
}
} else println("Images missing across translations: (Hidden)")
Expand Down Expand Up @@ -766,9 +770,14 @@ class LessonDownloader(
println("(look at above output for specific images that require verification).")
}

val hasAnyFailure = issues.isNotEmpty() || imageDownloadFailures.isNotEmpty() || renamedImages.isNotEmpty() || convertedImages.isNotEmpty()
val hasAnyFailure = (
issues.isNotEmpty() || imageDownloadFailures.isNotEmpty() ||
renamedImages.isNotEmpty() || convertedImages.isNotEmpty()
)
if (hasAnyFailure && failOnError) {
throw Exception("Failed to cleanly download and convert all lessons and images. See failures above.")
throw Exception(
"Failed to cleanly download and convert all lessons and images. See failures above."
)
}

// val translationMetrics = analyzer.computeTranslationsUsageReport()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.I
import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.ITEM_SELECTION_INPUT
import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.MATH_EQUATION_INPUT
import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.MULTIPLE_CHOICE_INPUT
import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.NUMBER_WITH_UNITS_INPUT
import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.NUMERIC_EXPRESSION_INPUT
import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.NUMERIC_INPUT
import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.RATIO_EXPRESSION_INPUT
Expand All @@ -89,6 +90,8 @@ import org.oppia.proto.v1.structure.MathEquationInputInstanceDto
import org.oppia.proto.v1.structure.MisconceptionDto
import org.oppia.proto.v1.structure.MultipleChoiceInputInstanceDto
import org.oppia.proto.v1.structure.NormalizedPoint2dDto
import org.oppia.proto.v1.structure.NumberWithUnitsDto
import org.oppia.proto.v1.structure.NumberWithUnitsInputInstanceDto
import org.oppia.proto.v1.structure.NumericExpressionInputInstanceDto
import org.oppia.proto.v1.structure.NumericInputInstanceDto
import org.oppia.proto.v1.structure.OutcomeDto
Expand All @@ -112,13 +115,10 @@ import org.oppia.proto.v1.structure.AlgebraicExpressionInputInstanceDto.RuleSpec
import org.oppia.proto.v1.structure.DragAndDropSortInputInstanceDto.RuleSpecDto as DragDropSortRuleSpecDto
import org.oppia.proto.v1.structure.DragAndDropSortInputInstanceDto.RuleSpecDto.RuleTypeCase as DragDropSortRuleSpecTypeCase
import org.oppia.proto.v1.structure.FractionInputInstanceDto.RuleSpecDto as FractionRuleSpecDto
import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.NUMBER_WITH_UNITS_INPUT
import org.oppia.proto.v1.structure.ItemSelectionInputInstanceDto.RuleSpecDto as ItemSelRuleSpecDto
import org.oppia.proto.v1.structure.MathEquationInputInstanceDto.RuleSpecDto as MathEqRuleSpecDto
import org.oppia.proto.v1.structure.MathEquationInputInstanceDto.RuleSpecDto.MatchesUpToTrivialManipulationsSpecDto as MathEqMatchesUpToTrivialManipulationsSpecDto
import org.oppia.proto.v1.structure.MultipleChoiceInputInstanceDto.RuleSpecDto as MultChoiceRuleSpecDto
import org.oppia.proto.v1.structure.NumberWithUnitsDto
import org.oppia.proto.v1.structure.NumberWithUnitsInputInstanceDto
import org.oppia.proto.v1.structure.NumberWithUnitsInputInstanceDto.RuleSpecDto as NumUnitsRuleSpecDto
import org.oppia.proto.v1.structure.NumericExpressionInputInstanceDto.RuleSpecDto as NumExpRuleSpecDto
import org.oppia.proto.v1.structure.NumericExpressionInputInstanceDto.RuleSpecDto.MatchesUpToTrivialManipulationsSpecDto as NumExpMatchesUpToTrivialManipulationsSpecDto
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,9 @@ class GaeAndroidEndpointJsonImpl(
return CoroutineScope(coroutineDispatcher).async {
SUPPORTED_CLASSROOMS.map { classroomName ->
CoroutineScope(coroutineDispatcher).async {
val classroomResult = activityService.fetchLatestClassroomAsync(classroomName).await().also {
val classroomResult = activityService.fetchLatestClassroomAsync(
classroomName
).await().also {
tracker.reportDownloaded(classroomName)
}
checkNotNull(classroomResult?.payload) { "Failed to fetch classroom: $classroomName." }
Expand Down Expand Up @@ -830,7 +832,7 @@ class GaeAndroidEndpointJsonImpl(
"Continue", "FractionInput", "ItemSelectionInput", "MultipleChoiceInput",
"NumericInput", "TextInput", "DragAndDropSortInput", "ImageClickInput",
"RatioExpressionInput", "EndExploration", "NumericExpressionInput",
"AlgebraicExpressionInput", "MathEquationInput"//, "NumberWithUnits"
"AlgebraicExpressionInput", "MathEquationInput" // , "NumberWithUnits"
)

// TODO: Remove gif and png since we only want to use svg(z) and webp moving forward.
Expand Down Expand Up @@ -862,7 +864,7 @@ class GaeAndroidEndpointJsonImpl(
private val SUPPORTED_DEFAULT_LANGUAGES = setOf(LanguageType.ENGLISH)

// From feconf.
private const val SUPPORTED_STATE_SCHEMA_VERSION = 56
private const val SUPPORTED_STATE_SCHEMA_VERSION = 57

private fun ClientCompatibilityContextDto.verifyCompatibility() {
check(topicListRequestResponseProtoVersion == createLatestTopicListProtoVersion()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ import org.oppia.android.scripts.gae.json.GaeSubtopicPage
import org.oppia.android.scripts.gae.json.GaeSubtopicPageContents
import org.oppia.android.scripts.gae.json.GaeTopic
import org.oppia.android.scripts.gae.json.GaeTranslatedContent
import org.oppia.android.scripts.gae.json.GaeWorkedExample
import org.oppia.android.scripts.gae.json.GaeWrittenTranslation
import org.oppia.android.scripts.gae.json.GaeWrittenTranslations
import org.oppia.android.scripts.gae.json.VersionedStructure
Expand Down Expand Up @@ -259,20 +258,11 @@ class StructureCompatibilityChecker(
defaultLanguage: LanguageType
): List<CompatibilityFailure> {
return gaeSkillContents.explanation.checkHasValidHtml(origin) +
gaeSkillContents.workedExamples.flatMap { checkWorkedExampleCompatibility(origin, it) } +
checkWrittenTranslationsCompatibility(
origin, gaeSkillContents.writtenTranslations, expectedTranslatedContentIds, defaultLanguage
) + checkRecordedVoiceoversCompatibility(origin, gaeSkillContents.recordedVoiceovers)
}

private fun checkWorkedExampleCompatibility(
origin: ContainerId,
gaeWorkedExample: GaeWorkedExample
): List<CompatibilityFailure> {
return gaeWorkedExample.question.checkHasValidHtml(origin) +
gaeWorkedExample.explanation.checkHasValidHtml(origin)
}

private fun checkWrittenTranslationsCompatibility(
origin: ContainerId,
gaeWrittenTranslations: GaeWrittenTranslations,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import org.oppia.android.scripts.gae.json.GaeSubtopic
import org.oppia.android.scripts.gae.json.GaeSubtopicPage
import org.oppia.android.scripts.gae.json.GaeTopic
import org.oppia.android.scripts.gae.json.GaeTranslatedContent
import org.oppia.android.scripts.gae.json.GaeWorkedExample
import org.oppia.android.scripts.gae.json.GaeWrittenTranslation
import org.oppia.android.scripts.gae.json.GaeWrittenTranslations
import org.oppia.android.scripts.gae.proto.LocalizationTracker
Expand Down Expand Up @@ -143,14 +142,10 @@ class SubtitledHtmlCollector(private val localizationTracker: LocalizationTracke

private fun GaeSkillContents.collectSubtitles(): Set<SubtitledText> {
val explanationText = setOf(explanation.toSubtitle())
val workedExampleTexts = workedExamples.flatSet { it.collectSubtitles() }
val translations = writtenTranslations.collectSubtitles()
return explanationText + workedExampleTexts + translations
return explanationText + translations
}

private fun GaeWorkedExample.collectSubtitles(): Set<SubtitledText> =
setOf(question.toSubtitle(), explanation.toSubtitle())

private fun GaeWrittenTranslations.collectSubtitles(): Set<SubtitledText> {
return translationsMapping.values.flatSet {
it.values.flatSet { translation -> translation.collectSubtitles() }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -623,9 +623,12 @@ private sealed class VersionedStructureReference<I : StructureId, S> {
Compatible -> LoadResult.Success(payload)
is Incompatible -> {
// TODO: Remove this once Oppia web supports pulling structures without schema migrations (see https://github.com/oppia/oppia/issues/21253).
val irrecoverableFailure = compatibilityResult.failures.firstOrNull { it is CompatibilityFailure.StateSchemaVersionTooNew }
val irrecoverableFailure = compatibilityResult.failures.firstOrNull {
it is CompatibilityFailure.StateSchemaVersionTooNew
}
check(irrecoverableFailure == null) {
"Oppia web introduced a state schema upgrade. Cannot recover--this script needs to be updated: $irrecoverableFailure"
"Oppia web introduced a state schema upgrade. Cannot recover--this " +
"script needs to be updated: $irrecoverableFailure"
}
LoadResult.Failure<S>(compatibilityResult.failures)
}
Expand Down Expand Up @@ -792,7 +795,9 @@ private class ExplorationFetcher(
): Deferred<VersionedStructure<CompleteExploration>?> {
return CoroutineScope(coroutineDispatcher).async {
val latestExp = service.fetchLatestExplorationAsync(id.id).await()
return@async latestExp?.let { it.copyWithNewPayload(service.downloadExploration(id, it.payload)) }
return@async latestExp?.let {
it.copyWithNewPayload(service.downloadExploration(id, it.payload))
}
}
}

Expand All @@ -803,7 +808,9 @@ private class ExplorationFetcher(
): Deferred<VersionedStructure<CompleteExploration>?> {
return CoroutineScope(coroutineDispatcher).async {
val latestExp = service.fetchSingleExplorationAsync(id.id, version).await()
return@async latestExp?.let { it.copyWithNewPayload(service.downloadExploration(id, it.payload)) }
return@async latestExp?.let {
it.copyWithNewPayload(service.downloadExploration(id, it.payload))
}
}
}

Expand Down Expand Up @@ -833,7 +840,9 @@ private class ExplorationFetcher(
val missingTranslations = translations.mapIndexedNotNull { index, value ->
index.takeIf { value == null } // Take only indexes corresponding to missing values.
}.map { VALID_LANGUAGE_TYPES[it] }
check(missingTranslations.isEmpty()) { "Failed to fetch translations for exploration $id: $missingTranslations." }
check(missingTranslations.isEmpty()) {
"Failed to fetch translations for exploration $id: $missingTranslations."
}
return CompleteExploration(
exploration,
receivedTranslations.associateBy {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,10 @@ class AndroidActivityHandlerService(
val metadata = RequestMetadata(request().method, request().url.toUrl().toExternalForm())
val responseBodyText = memoizedRawResponses[metadata]
if (responseBodyText?.contains("maxmemory") == true) {
throw IllegalStateException("Oppia web ran out of memory. Clear Redis memory cache and try again.", exception)
throw IllegalStateException(
"Oppia web ran out of memory. Clear Redis memory cache and try again.",
exception
)
} else throw IllegalStateException(
"Failed to call: ${request()}. Response body:\n\n$responseBodyText".redact(), exception
)
Expand Down Expand Up @@ -370,7 +373,9 @@ class AndroidActivityHandlerService(
type: String,
id: String,
version: Int,
crossinline fetch: (AndroidActivityRequests.SingleNonLocalized) -> Call<List<VersionedStructure<T>>>,
crossinline fetch: (
AndroidActivityRequests.SingleNonLocalized
) -> Call<List<VersionedStructure<T>>>,
noinline retrieveStructureVersion: ((T) -> Int)?
): Deferred<VersionedStructure<T>?> {
return CoroutineScope(dispatcher).async {
Expand Down Expand Up @@ -423,7 +428,10 @@ class AndroidActivityHandlerService(
val reqsCol = createRequests(requestsRequiringRemoteFetching.map { (_, req) -> req })
val fetchResult = if (reqsCol.requests.isNotEmpty()) {
// Only fetch if there are versions to retrieve.
val versionsToRetrieve = requestsRequiringRemoteFetching.map { (index, _) -> versions[index] }
val versionsToRetrieve = requestsRequiringRemoteFetching.map {
(index, _) ->
versions[index]
}
fetch(reqsCol).resolveAsyncVersionsAsync(id, versionsToRetrieve).await().map { structure ->
// Ensure that the returned structures have the correct remote versions (since the web
// controller isn't consistent in when it provides a version).
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@ import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.I
import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.ITEM_SELECTION_INPUT
import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.MATH_EQUATION_INPUT
import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.MULTIPLE_CHOICE_INPUT
import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.NUMBER_WITH_UNITS_INPUT
import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.NUMERIC_EXPRESSION_INPUT
import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.NUMERIC_INPUT
import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.RATIO_EXPRESSION_INPUT
import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.TEXT_INPUT
import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.NUMBER_WITH_UNITS_INPUT

// TODO: Mention parsing this requires setting customization key name & interaction type in the parsing context.
sealed class GaeCustomizationArgValue {
Expand Down
Loading
Loading