From 7b2fd3576a88c0ee5a1c83a39fe451960bebcab9 Mon Sep 17 00:00:00 2001 From: DO YUNG YOON Date: Tue, 25 Dec 2018 20:39:55 +0900 Subject: [PATCH 01/13] add HFileParserUDF. --- .../s2jobs/loader/HFileGenerator.scala | 7 +- .../apache/s2graph/s2jobs/task/Source.scala | 17 +- .../s2jobs/wal/udfs/HFileParserUDF.scala | 69 +++++ .../s2jobs/wal/udfs/IsEmptyByteArrayUDF.scala | 16 + .../s2jobs/wal/utils/DeserializeUtil.scala | 276 ++++++++++++++++++ .../s2graph/s2jobs/wal/utils/SchemaUtil.scala | 95 ++++++ .../s2jobs/wal/utils/TaskConfUtil.scala | 18 ++ .../s2jobs/wal/udfs/HFileParserUDFTest.scala | 45 +++ 8 files changed, 532 insertions(+), 11 deletions(-) create mode 100644 s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDF.scala create mode 100644 s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/IsEmptyByteArrayUDF.scala create mode 100644 s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala create mode 100644 s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala create mode 100644 s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/TaskConfUtil.scala create mode 100644 s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/HFileGenerator.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/HFileGenerator.scala index 3d3821e4..ed5a75e8 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/HFileGenerator.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/HFileGenerator.scala @@ -167,13 +167,13 @@ object HFileGenerator extends RawFileGenerator[String, KeyValue] { columnFamily: String = "e", batchSize: Int = 1000, labelMapping: Map[String, String] = Map.empty, - buildDegree: Boolean = false): RDD[Seq[Cell]] = { + buildDegree: Boolean = false): RDD[(ImmutableBytesWritable, Result)] = { val cf = Bytes.toBytes(columnFamily) val hbaseConfig = HBaseConfiguration.create(ss.sparkContext.hadoopConfiguration) hbaseConfig.set("hbase.rootdir", snapshotPath) - val initial = ss.sparkContext.parallelize(Seq.empty[Seq[Cell]]) + val initial = ss.sparkContext.parallelize(Seq.empty[(ImmutableBytesWritable, Result)]) tableNames.foldLeft(initial) { case (prev, tableName) => val scan = new Scan scan.addFamily(cf) @@ -186,7 +186,8 @@ object HFileGenerator extends RawFileGenerator[String, KeyValue] { val current = ss.sparkContext.newAPIHadoopRDD(job.getConfiguration, classOf[TableSnapshotInputFormat], classOf[ImmutableBytesWritable], - classOf[Result]).map(_._2.listCells().asScala.toSeq) + classOf[Result]) +// .map(_._2.listCells().asScala.toSeq) prev ++ current } diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala index 8e4e234c..db04088a 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala @@ -24,6 +24,7 @@ import org.apache.s2graph.s2jobs.Schema import org.apache.s2graph.s2jobs.loader.{HFileGenerator, SparkBulkLoaderTransformer} import org.apache.s2graph.s2jobs.serde.reader.S2GraphCellReader import org.apache.s2graph.s2jobs.serde.writer.RowDataFrameWriter +import org.apache.s2graph.s2jobs.wal.utils.DeserializeUtil import org.apache.spark.sql.{DataFrame, SparkSession} import play.api.libs.json.{JsObject, Json} @@ -133,6 +134,8 @@ class HiveSource(conf:TaskConf) extends Source(conf) { class S2GraphSource(conf: TaskConf) extends Source(conf) { import org.apache.s2graph.spark.sql.streaming.S2SourceConfigs._ + import scala.collection.JavaConverters._ + override def mandatoryOptions: Set[String] = Set( S2_SOURCE_BULKLOAD_HBASE_ROOT_DIR, S2_SOURCE_BULKLOAD_RESTORE_PATH, @@ -154,18 +157,16 @@ class S2GraphSource(conf: TaskConf) extends Source(conf) { val buildDegree = if (columnFamily == "v") false else conf.options.getOrElse(S2_SOURCE_BULKLOAD_BUILD_DEGREE, "false").toBoolean - val elementType = conf.options.getOrElse(S2_SOURCE_ELEMENT_TYPE, "IndexEdge") - val schema = if (columnFamily == "v") Schema.VertexSchema else Schema.EdgeSchema +// val elementType = conf.options.getOrElse(S2_SOURCE_ELEMENT_TYPE, "IndexEdge") +// val schema = if (columnFamily == "v") Schema.VertexSchema else Schema.EdgeSchema val cells = HFileGenerator.tableSnapshotDump(ss, config, snapshotPath, restorePath, tableNames, columnFamily, batchSize, labelMapping, buildDegree) - implicit val reader = new S2GraphCellReader(elementType) - implicit val writer = new RowDataFrameWriter - - val transformer = new SparkBulkLoaderTransformer(config, labelMapping, buildDegree) - val kvs = transformer.transform(cells) + val skvs = cells.flatMap { case (_, result) => + result.listCells().asScala.map(DeserializeUtil.cellToSKeyValue) + } - ss.sqlContext.createDataFrame(kvs, schema) + ss.createDataFrame(skvs) } } \ No newline at end of file diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDF.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDF.scala new file mode 100644 index 00000000..3a587e63 --- /dev/null +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDF.scala @@ -0,0 +1,69 @@ +package org.apache.s2graph.s2jobs.wal.udfs + +import com.typesafe.config.ConfigFactory +import org.apache.s2graph.core.Management +import org.apache.s2graph.core.schema.{Label, LabelIndex, LabelMeta} +import org.apache.s2graph.core.types.HBaseType +import org.apache.s2graph.s2jobs.udfs.Udf +import org.apache.s2graph.s2jobs.wal.utils.SchemaUtil +import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.functions.udf + + +object HFileParserUDF { + + import org.apache.s2graph.s2jobs.wal.utils.DeserializeUtil._ + + def indexEdgeParser(labelServices: Map[Int, String], + labels: Map[Int, Label], + labelIndices: Map[Int, Map[Byte, LabelIndex]], + labelMetas: Map[Int, Map[Byte, LabelMeta]], + labelIndexLabelMetas: Map[Int, Map[Byte, Array[LabelMeta]]], + tallSchemaVersions: Set[String]) = { + udf((row: Row) => { + val skv = sKeyValueFromRow(row) + + indexEdgeKeyValueToRow(skv, None, labelServices, labels, labelIndices, labelMetas, labelIndexLabelMetas, tallSchemaVersions) + }) + } + + def snapshotEdgeParser(labelServices: Map[Int, String], + labels: Map[Int, Label], + labelMetas: Map[Int, Map[Byte, LabelMeta]]) = { + udf((row: Row) => { + val skv = sKeyValueFromRow(row) + snapshotEdgeKeyValueToRow(skv, None, labelServices, labels, labelMetas) + }) + } +} + +class HFileParserUDF extends Udf { + + import HFileParserUDF._ + import org.apache.s2graph.s2jobs.wal.utils.TaskConfUtil._ + + override def register(ss: SparkSession, name: String, options: Map[String, String]): Unit = { + val mergedConfig = Management.toConfig(parseMetaStoreConfigs(options)) + val config = ConfigFactory.load(mergedConfig) + SchemaUtil.init(config) + + val elementType = options.getOrElse("elementType", "indexedge") + val labelNames = options.get("labelNames").map(_.split(",").toSeq).getOrElse(Nil) + + val (labelServices, labels, labelIndices, labelMetas, labelIndexLabelMetas) = SchemaUtil.buildSchemaForLabels(labelNames) + + // logger.error(s"LabelServices: $labelServices") + // logger.error(s"Labels: $labels") + // logger.error(s"LabelIndices: $labelIndices") + // logger.error(s"LabelMetas: $labelMetas") + val tallSchemaVersions = Set(HBaseType.VERSION4) + + val f = elementType.toLowerCase match { + case "indexedge" => indexEdgeParser(labelServices, labels, labelIndices, labelMetas, labelIndexLabelMetas, tallSchemaVersions) + case "snapshotedge" => snapshotEdgeParser(labelServices, labels, labelMetas) + case _ => throw new IllegalArgumentException(s"$elementType is not supported.") + } + + ss.udf.register(name, f) + } +} \ No newline at end of file diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/IsEmptyByteArrayUDF.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/IsEmptyByteArrayUDF.scala new file mode 100644 index 00000000..3c426eaf --- /dev/null +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/IsEmptyByteArrayUDF.scala @@ -0,0 +1,16 @@ +package org.apache.s2graph.s2jobs.wal.udfs + +import org.apache.s2graph.s2jobs.udfs.Udf +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.functions.udf + + +class IsEmptyByteArrayUDF extends Udf { + override def register(ss: SparkSession, name: String, options: Map[String, String]): Unit = { + val f = udf((bytes: Array[Byte]) => { + bytes.isEmpty + }) + + ss.udf.register(name, f) + } +} diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala new file mode 100644 index 00000000..da255415 --- /dev/null +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala @@ -0,0 +1,276 @@ +package org.apache.s2graph.s2jobs.wal.utils + +import org.apache.hadoop.hbase.Cell +import org.apache.hadoop.hbase.util.Bytes +import org.apache.s2graph.core.{GraphUtil, JSONParser, SnapshotEdge} +import org.apache.s2graph.core.schema.{Label, LabelIndex, LabelMeta, ServiceColumn} +import org.apache.s2graph.core.storage.SKeyValue +import org.apache.s2graph.core.storage.serde.StorageDeserializable +import org.apache.s2graph.core.types._ +import org.apache.s2graph.s2jobs.S2GraphHelper.logger +import org.apache.s2graph.s2jobs.wal.WalLog +import org.apache.spark.sql.Row +import play.api.libs.json.Json + + +object DeserializeUtil { + + private def statusCodeWithOp(byte: Byte): (Byte, Byte) = { + val statusCode = byte >> 4 + val op = byte & ((1 << 4) - 1) + (statusCode.toByte, op.toByte) + } + + private def bytesToLabelIndexSeqWithIsInverted(bytes: Array[Byte], offset: Int): (Byte, Boolean) = { + val byte = bytes(offset) + val isInverted = if ((byte & 1) != 0) true else false + val labelOrderSeq = byte >> 1 + (labelOrderSeq.toByte, isInverted) + } + + private def bytesToKeyValues(bytes: Array[Byte], + offset: Int, + length: Int, + schemaVer: String, + labelMetaMap: Map[Byte, LabelMeta]): (Array[(LabelMeta, InnerValLike)], Int) = { + var pos = offset + val len = bytes(pos) + pos += 1 + val kvs = new Array[(LabelMeta, InnerValLike)](len) + var i = 0 + while (i < len) { + val k = labelMetaMap(bytes(pos)) + pos += 1 + val (v, numOfBytesUsed) = InnerVal.fromBytes(bytes, pos, 0, schemaVer) + pos += numOfBytesUsed + kvs(i) = (k -> v) + i += 1 + } + val ret = (kvs, pos) + // logger.debug(s"bytesToProps: $ret") + ret + } + + private def bytesToKeyValuesWithTs(bytes: Array[Byte], + offset: Int, + schemaVer: String, + labelMetaMap: Map[Byte, LabelMeta]): (Array[(LabelMeta, InnerValLikeWithTs)], Int) = { + var pos = offset + val len = bytes(pos) + pos += 1 + val kvs = new Array[(LabelMeta, InnerValLikeWithTs)](len) + var i = 0 + while (i < len) { + val k = labelMetaMap(bytes(pos)) + pos += 1 + val (v, numOfBytesUsed) = InnerValLikeWithTs.fromBytes(bytes, pos, 0, schemaVer) + pos += numOfBytesUsed + kvs(i) = (k -> v) + i += 1 + } + val ret = (kvs, pos) + // logger.debug(s"bytesToProps: $ret") + ret + } + + def sKeyValueFromRow(row: Row): SKeyValue = { + val table = row.getAs[Array[Byte]]("table") + val _row = row.getAs[Array[Byte]]("row") + val cf = row.getAs[Array[Byte]]("cf") + val qualifier = row.getAs[Array[Byte]]("qualifier") + val value = row.getAs[Array[Byte]]("value") + val timestamp = row.getAs[Long]("timestamp") + val operation = row.getAs[Int]("operation") + val durability = row.getAs[Boolean]("durability") + + SKeyValue(table, _row, cf, qualifier, value, timestamp, operation, durability) + } + + def cellToSKeyValue(cell: Cell): SKeyValue = { + new SKeyValue(Array.empty[Byte], cell.getRow, cell.getFamily, cell.getQualifier, + cell.getValue, cell.getTimestamp, SKeyValue.Default) + } + + def indexEdgeKeyValueToRow(kv: SKeyValue, + cacheElementOpt: Option[SnapshotEdge], + labelServices: Map[Int, String], + labels: Map[Int, Label], + labelIndices: Map[Int, Map[Byte, LabelIndex]], + labelMetas: Map[Int, Map[Byte, LabelMeta]], + labelIndexLabelMetas: Map[Int, Map[Byte, Array[LabelMeta]]], + tallSchemaVersions: Set[String]): Option[WalLog] = { + try { + var pos = 0 + val (srcVertexId, srcIdLen) = SourceVertexId.fromBytes(kv.row, pos, kv.row.length, HBaseType.DEFAULT_VERSION) + pos += srcIdLen + val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4)) + pos += 4 + val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(kv.row, pos) + pos += 1 + + if (isInverted) None + else if (!labels.contains(labelWithDir.labelId)) None + else { + val label = labels.getOrElse(labelWithDir.labelId, throw new IllegalArgumentException(s"$labelWithDir labelId is not found.")) + val schemaVer = label.schemaVersion + + val isTallSchema = tallSchemaVersions(label.schemaVersion) + val isDegree = if (isTallSchema) pos == kv.row.length else kv.qualifier.isEmpty + + if (isDegree) { + None + } else { + // not degree edge + val (idxPropsRaw, endAt) = + if (isTallSchema) StorageDeserializable.bytesToProps(kv.row, pos, schemaVer) + else { + StorageDeserializable.bytesToProps(kv.qualifier, 0, schemaVer) + } + pos = endAt + + val (tgtVertexIdRaw, tgtVertexIdLen) = if (isTallSchema) { + if (endAt == kv.row.length - 1) { + (HBaseType.defaultTgtVertexId, 0) + } else { + TargetVertexId.fromBytes(kv.row, endAt, kv.row.length - 1, schemaVer) + } + } else { + if (endAt == kv.qualifier.length) { + (HBaseType.defaultTgtVertexId, 0) + } else { + TargetVertexId.fromBytes(kv.qualifier, endAt, kv.qualifier.length, schemaVer) + } + } + pos += tgtVertexIdLen + + val op = + if (isTallSchema) kv.row(kv.row.length - 1) + else { + if (kv.qualifier.length == pos) GraphUtil.defaultOpByte + else kv.qualifier(kv.qualifier.length - 1) + } + + val indices = labelIndices.getOrElse(labelWithDir.labelId, throw new IllegalArgumentException(s"labelIndices not found. ${labelWithDir}")) + val index = indices.getOrElse(labelIdxSeq, throw new IllegalArgumentException(s"invalid index seq: ${label.id.get}, ${labelIdxSeq}")) + val indexLabelMetas = labelIndexLabelMetas.getOrElse(labelWithDir.labelId, throw new IllegalArgumentException(s"${labelWithDir.labelId} not found in labelIndexLabelMetas")) + val sortKeyTypesArray = indexLabelMetas.getOrElse(labelIdxSeq, throw new IllegalArgumentException(s"invalid index seq for meta array: ${label.id.get}, ${labelIdxSeq}")) + + /* process indexProps */ + val size = idxPropsRaw.length + val idxProps = (0 until size).map { ith => + val meta = sortKeyTypesArray(ith) + val (k, v) = idxPropsRaw(ith) + meta -> v + } + + /* process props */ + val metaProps = + if (op == GraphUtil.operations("incrementCount")) { + // val countVal = Bytes.toLong(kv.value) + val countVal = StorageDeserializable.bytesToLong(kv.value, 0) + Array(LabelMeta.count -> InnerVal.withLong(countVal, schemaVer)) + } else { + val (props, endAt) = bytesToKeyValues(kv.value, 0, kv.value.length, schemaVer, labelMetas(labelWithDir.labelId)) + props + } + + val mergedProps = (idxProps ++ metaProps).toMap + val tsInnerVal = mergedProps(LabelMeta.timestamp) + val propsJson = for { + (labelMeta, innerValLike) <- mergedProps + jsValue <- JSONParser.innerValToJsValue(innerValLike, labelMeta.dataType) + } yield { + labelMeta.name -> jsValue + } + /* process tgtVertexId */ + // skip + val tgtVertexIdInnerId = mergedProps.getOrElse(LabelMeta.to, tgtVertexIdRaw.innerId) + + val wal = WalLog( + tsInnerVal.toIdString().toLong, + GraphUtil.fromOp(op), + "edge", + srcVertexId.innerId.toIdString(), + tgtVertexIdInnerId.toIdString(), + labelServices(labelWithDir.labelId), + label.label, + Json.toJson(propsJson).toString() + ) + + Option(wal) + } + } + } catch { + case e: Exception => + println(e.toString) + logger.error(s"$kv", e) + None + } + } + + def snapshotEdgeKeyValueToRow(kv: SKeyValue, + cacheElementOpt: Option[SnapshotEdge], + labelServices: Map[Int, String], + labelsMap: Map[Int, Label], + labelMetasMap: Map[Int, Map[Byte, LabelMeta]]): Option[WalLog] = { + try { + var pos = 0 + val (srcVertexId, srcIdLen) = SourceVertexId.fromBytes(kv.row, pos, kv.row.length, HBaseType.DEFAULT_VERSION) + pos += srcIdLen + + val isTallSchema = pos + 5 != kv.row.length + var tgtVertexId = TargetVertexId(ServiceColumn.Default, srcVertexId.innerId) + + if (isTallSchema) { + val (tgtId, tgtBytesLen) = InnerVal.fromBytes(kv.row, pos, kv.row.length, HBaseType.DEFAULT_VERSION) + tgtVertexId = TargetVertexId(ServiceColumn.Default, tgtId) + pos += tgtBytesLen + } + + val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4)) + pos += 4 + val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(kv.row, pos) + pos += 1 + + if (!isInverted) None + else { + val label = labelsMap(labelWithDir.labelId) + val schemaVer = label.schemaVersion + + var pos = 0 + val (statusCode, op) = statusCodeWithOp(kv.value(pos)) + pos += 1 + val (props, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer, labelMetasMap(labelWithDir.labelId)) + val kvsMap = props.toMap + val tsInnerVal = kvsMap(LabelMeta.timestamp).innerVal + + val propsJson = for { + (labelMeta, innerValLikeWithTs) <- props + jsValue <- JSONParser.innerValToJsValue(innerValLikeWithTs.innerVal, labelMeta.dataType) + } yield { + labelMeta.name -> jsValue + } + + pos = endAt + + val wal = WalLog( + tsInnerVal.toIdString().toLong, + GraphUtil.fromOp(op), + "edge", + srcVertexId.innerId.toIdString(), + tgtVertexId.innerId.toIdString(), + labelServices(labelWithDir.labelId), + label.label, + Json.toJson(propsJson.toMap).toString() + ) + + Option(wal) + } + } catch { + case e: Exception => + println(e.toString) + logger.error(e) + None + } + } +} diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala new file mode 100644 index 00000000..f0d71313 --- /dev/null +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala @@ -0,0 +1,95 @@ +package org.apache.s2graph.s2jobs.wal.utils + +import com.typesafe.config.Config +import org.apache.s2graph.core.schema.{Label, LabelMeta, Schema} +import org.apache.s2graph.core.utils.logger + + +object SchemaUtil { + + import scalikejdbc._ + + var initialized = false + + def init(config: Config): Boolean = { + synchronized { + if (!initialized) { + logger.info(s"[SCHEMA]: initializing...") + logger.info(s"[SCHEMA]: $config") + Schema.apply(config) + initialized = true + } + + initialized + } + } + + def labelSrcColumnMap: Map[String, String] = { + val ls = DB readOnly { implicit session => + sql"""SELECT label, src_column_name FROM labels""".map { rs => + rs.string("label") -> rs.string("src_column_name") + }.list().apply() + } + + ls.toMap + } + + def parseLabelMeta(rs: WrappedResultSet): LabelMeta = { + LabelMeta( + rs.intOpt("id"), + rs.int("label_id"), + rs.string("name"), + rs.byte("seq"), + rs.string("default_value"), + rs.string("data_type").toLowerCase, + false + // rs.boolean("store_in_global_index") + ) + } + + def labelMetaMap(label: Label): Map[Byte, LabelMeta] = { + val reserved = LabelMeta.reservedMetas.map { m => + if (m == LabelMeta.to) m.copy(dataType = label.tgtColumnType) + else if (m == LabelMeta.from) m.copy(dataType = label.srcColumnType) + else m + } + + val labelMetas = DB readOnly { implicit session => + sql"""SELECT * FROM label_metas WHERE label_id = ${label.id.get}""".map(parseLabelMeta).list().apply() + } + + (reserved ++ labelMetas).groupBy(_.seq).map { case (seq, labelMetas) => + seq -> labelMetas.head + } + } + + def buildSchemaForLabels(labelNames: Seq[String]) = { + val labels = labelNames.map { labelName => + val label = Label.findByName(labelName).getOrElse(throw new IllegalArgumentException(s"$labelName not exist.")) + label.id.get -> label + }.toMap + + val labelServices = labels.map { case (id, label) => + id -> label.serviceName + } + + val labelIndices = labels.map { case (id, label) => + id -> label.indicesMap + } + + val labelMetas = labels.map { case (id, label) => + id -> labelMetaMap(labels(id)) + } + + val labelIndexLabelMetas = labelIndices.map { case (id, indicesMap) => + val metaPropsMap = labelMetas(id) + val m = indicesMap.map { case (idxSeq, labelIndex) => + idxSeq -> labelIndex.metaSeqs.flatMap(seq => metaPropsMap.get(seq)).toArray + } + + id -> m + } + + (labelServices, labels, labelIndices, labelMetas, labelIndexLabelMetas) + } +} diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/TaskConfUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/TaskConfUtil.scala new file mode 100644 index 00000000..3b4d73c5 --- /dev/null +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/TaskConfUtil.scala @@ -0,0 +1,18 @@ +package org.apache.s2graph.s2jobs.wal.utils + +import org.apache.s2graph.core.S2GraphConfigs + +object TaskConfUtil { + + def parseHBaseConfigs(options: Map[String, String]): Map[String, Any] = { + options.filterKeys(S2GraphConfigs.HBaseConfigs.DEFAULTS.keySet) + } + + def parseMetaStoreConfigs(options: Map[String, String]): Map[String, Any] = { + options.filterKeys(S2GraphConfigs.DBConfigs.DEFAULTS.keySet) + } + + def parseLocalCacheConfigs(options: Map[String, String]): Map[String, Any] = { + options.filterKeys(S2GraphConfigs.CacheConfigs.DEFAULTS.keySet).mapValues(_.toInt) + } +} diff --git a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala new file mode 100644 index 00000000..a5ed70df --- /dev/null +++ b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala @@ -0,0 +1,45 @@ +package org.apache.s2graph.s2jobs.wal.udfs + +import com.holdenkarau.spark.testing.DataFrameSuiteBase +import org.apache.spark.sql.SaveMode +import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers} + +class HFileParserUDFTest extends FunSuite with Matchers with BeforeAndAfterAll with DataFrameSuiteBase { + + test("deserialize result udf.") { + + val sKeyValueParser = new HFileParserUDF + sKeyValueParser.register(spark, "to_edge", Map( + "elementType" -> "indexedge", + "labelNames" -> "talk_friend", + "db.default.url" -> "jdbc:mysql://comm-s2graph.mydb.iwilab.com/graph", + "db.default.driver" -> "com.mysql.jdbc.Driver", + "db.default.user" -> "graph", + "db.default.password" -> "graph" + )) + + val df = spark.read.parquet("test.parquet").limit(100) + df.printSchema() + + df.createOrReplaceTempView("kvs") + +// val sql = """select *, to_edge(struct(*)) as wal from kvs""" + val sql = + """ + |SELECT wal + |FROM ( + | SELECT to_edge(struct(*)) as wal + | FROM kvs + |) + |WHERE wal is not null + """.stripMargin + + val startedAt = System.currentTimeMillis() + val result = spark.sql(sql) + // println(s"[Count]: ${result.count()}") +// result.write.mode(SaveMode.Overwrite).json("./parsed.json") + result.show(false) + val duration = System.currentTimeMillis() - startedAt + println("[Duration]: " + duration) + } +} From 23793d47f1102fc23f7a07f4b2ac53d45e45e0ef Mon Sep 17 00:00:00 2001 From: DO YUNG YOON Date: Wed, 26 Dec 2018 11:00:30 +0900 Subject: [PATCH 02/13] add LabelSchema. --- .../s2graph/s2jobs/wal/LabelSchema.scala | 10 +++++++ .../s2jobs/wal/udfs/HFileParserUDF.scala | 21 ++++++--------- .../s2jobs/wal/utils/DeserializeUtil.scala | 26 ++++++++----------- .../s2graph/s2jobs/wal/utils/SchemaUtil.scala | 5 ++-- 4 files changed, 32 insertions(+), 30 deletions(-) create mode 100644 s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/LabelSchema.scala diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/LabelSchema.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/LabelSchema.scala new file mode 100644 index 00000000..d260b32f --- /dev/null +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/LabelSchema.scala @@ -0,0 +1,10 @@ +package org.apache.s2graph.s2jobs.wal + +import org.apache.s2graph.core.schema._ + + +case class LabelSchema(labelService: Map[Int, String], + labels: Map[Int, Label], + labelIndices: Map[Int, Map[Byte, LabelIndex]], + labelIndexLabelMetas: Map[Int, Map[Byte, Array[LabelMeta]]], + labelMetas: Map[Int, Map[Byte, LabelMeta]]) \ No newline at end of file diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDF.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDF.scala index 3a587e63..119115bd 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDF.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDF.scala @@ -5,6 +5,7 @@ import org.apache.s2graph.core.Management import org.apache.s2graph.core.schema.{Label, LabelIndex, LabelMeta} import org.apache.s2graph.core.types.HBaseType import org.apache.s2graph.s2jobs.udfs.Udf +import org.apache.s2graph.s2jobs.wal.LabelSchema import org.apache.s2graph.s2jobs.wal.utils.SchemaUtil import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.functions.udf @@ -14,25 +15,19 @@ object HFileParserUDF { import org.apache.s2graph.s2jobs.wal.utils.DeserializeUtil._ - def indexEdgeParser(labelServices: Map[Int, String], - labels: Map[Int, Label], - labelIndices: Map[Int, Map[Byte, LabelIndex]], - labelMetas: Map[Int, Map[Byte, LabelMeta]], - labelIndexLabelMetas: Map[Int, Map[Byte, Array[LabelMeta]]], + def indexEdgeParser(labelSchema: LabelSchema, tallSchemaVersions: Set[String]) = { udf((row: Row) => { val skv = sKeyValueFromRow(row) - indexEdgeKeyValueToRow(skv, None, labelServices, labels, labelIndices, labelMetas, labelIndexLabelMetas, tallSchemaVersions) + indexEdgeKeyValueToRow(skv, None, labelSchema, tallSchemaVersions) }) } - def snapshotEdgeParser(labelServices: Map[Int, String], - labels: Map[Int, Label], - labelMetas: Map[Int, Map[Byte, LabelMeta]]) = { + def snapshotEdgeParser(labelSchema: LabelSchema) = { udf((row: Row) => { val skv = sKeyValueFromRow(row) - snapshotEdgeKeyValueToRow(skv, None, labelServices, labels, labelMetas) + snapshotEdgeKeyValueToRow(skv, None, labelSchema) }) } } @@ -50,7 +45,7 @@ class HFileParserUDF extends Udf { val elementType = options.getOrElse("elementType", "indexedge") val labelNames = options.get("labelNames").map(_.split(",").toSeq).getOrElse(Nil) - val (labelServices, labels, labelIndices, labelMetas, labelIndexLabelMetas) = SchemaUtil.buildSchemaForLabels(labelNames) + val labelSchema = SchemaUtil.buildLabelSchema(labelNames) // logger.error(s"LabelServices: $labelServices") // logger.error(s"Labels: $labels") @@ -59,8 +54,8 @@ class HFileParserUDF extends Udf { val tallSchemaVersions = Set(HBaseType.VERSION4) val f = elementType.toLowerCase match { - case "indexedge" => indexEdgeParser(labelServices, labels, labelIndices, labelMetas, labelIndexLabelMetas, tallSchemaVersions) - case "snapshotedge" => snapshotEdgeParser(labelServices, labels, labelMetas) + case "indexedge" => indexEdgeParser(labelSchema, tallSchemaVersions) + case "snapshotedge" => snapshotEdgeParser(labelSchema) case _ => throw new IllegalArgumentException(s"$elementType is not supported.") } diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala index da255415..f52e96cc 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala @@ -8,7 +8,7 @@ import org.apache.s2graph.core.storage.SKeyValue import org.apache.s2graph.core.storage.serde.StorageDeserializable import org.apache.s2graph.core.types._ import org.apache.s2graph.s2jobs.S2GraphHelper.logger -import org.apache.s2graph.s2jobs.wal.WalLog +import org.apache.s2graph.s2jobs.wal.{LabelSchema, WalLog} import org.apache.spark.sql.Row import play.api.libs.json.Json @@ -93,13 +93,11 @@ object DeserializeUtil { def indexEdgeKeyValueToRow(kv: SKeyValue, cacheElementOpt: Option[SnapshotEdge], - labelServices: Map[Int, String], - labels: Map[Int, Label], - labelIndices: Map[Int, Map[Byte, LabelIndex]], - labelMetas: Map[Int, Map[Byte, LabelMeta]], - labelIndexLabelMetas: Map[Int, Map[Byte, Array[LabelMeta]]], + labelSchema: LabelSchema, tallSchemaVersions: Set[String]): Option[WalLog] = { try { + val LabelSchema(labelService, labels, labelIndices, labelIndexLabelMetas, labelMetas) = labelSchema + var pos = 0 val (srcVertexId, srcIdLen) = SourceVertexId.fromBytes(kv.row, pos, kv.row.length, HBaseType.DEFAULT_VERSION) pos += srcIdLen @@ -150,8 +148,6 @@ object DeserializeUtil { else kv.qualifier(kv.qualifier.length - 1) } - val indices = labelIndices.getOrElse(labelWithDir.labelId, throw new IllegalArgumentException(s"labelIndices not found. ${labelWithDir}")) - val index = indices.getOrElse(labelIdxSeq, throw new IllegalArgumentException(s"invalid index seq: ${label.id.get}, ${labelIdxSeq}")) val indexLabelMetas = labelIndexLabelMetas.getOrElse(labelWithDir.labelId, throw new IllegalArgumentException(s"${labelWithDir.labelId} not found in labelIndexLabelMetas")) val sortKeyTypesArray = indexLabelMetas.getOrElse(labelIdxSeq, throw new IllegalArgumentException(s"invalid index seq for meta array: ${label.id.get}, ${labelIdxSeq}")) @@ -192,7 +188,7 @@ object DeserializeUtil { "edge", srcVertexId.innerId.toIdString(), tgtVertexIdInnerId.toIdString(), - labelServices(labelWithDir.labelId), + labelService(labelWithDir.labelId), label.label, Json.toJson(propsJson).toString() ) @@ -210,10 +206,10 @@ object DeserializeUtil { def snapshotEdgeKeyValueToRow(kv: SKeyValue, cacheElementOpt: Option[SnapshotEdge], - labelServices: Map[Int, String], - labelsMap: Map[Int, Label], - labelMetasMap: Map[Int, Map[Byte, LabelMeta]]): Option[WalLog] = { + labelSchema: LabelSchema): Option[WalLog] = { try { + val LabelSchema(labelService, labels, _, _, labelMetas) = labelSchema + var pos = 0 val (srcVertexId, srcIdLen) = SourceVertexId.fromBytes(kv.row, pos, kv.row.length, HBaseType.DEFAULT_VERSION) pos += srcIdLen @@ -234,13 +230,13 @@ object DeserializeUtil { if (!isInverted) None else { - val label = labelsMap(labelWithDir.labelId) + val label = labels(labelWithDir.labelId) val schemaVer = label.schemaVersion var pos = 0 val (statusCode, op) = statusCodeWithOp(kv.value(pos)) pos += 1 - val (props, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer, labelMetasMap(labelWithDir.labelId)) + val (props, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer, labelMetas(labelWithDir.labelId)) val kvsMap = props.toMap val tsInnerVal = kvsMap(LabelMeta.timestamp).innerVal @@ -259,7 +255,7 @@ object DeserializeUtil { "edge", srcVertexId.innerId.toIdString(), tgtVertexId.innerId.toIdString(), - labelServices(labelWithDir.labelId), + labelService(labelWithDir.labelId), label.label, Json.toJson(propsJson.toMap).toString() ) diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala index f0d71313..0869f8ad 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala @@ -3,6 +3,7 @@ package org.apache.s2graph.s2jobs.wal.utils import com.typesafe.config.Config import org.apache.s2graph.core.schema.{Label, LabelMeta, Schema} import org.apache.s2graph.core.utils.logger +import org.apache.s2graph.s2jobs.wal.LabelSchema object SchemaUtil { @@ -63,7 +64,7 @@ object SchemaUtil { } } - def buildSchemaForLabels(labelNames: Seq[String]) = { + def buildLabelSchema(labelNames: Seq[String]): LabelSchema = { val labels = labelNames.map { labelName => val label = Label.findByName(labelName).getOrElse(throw new IllegalArgumentException(s"$labelName not exist.")) label.id.get -> label @@ -90,6 +91,6 @@ object SchemaUtil { id -> m } - (labelServices, labels, labelIndices, labelMetas, labelIndexLabelMetas) + LabelSchema(labelServices, labels, labelIndices, labelIndexLabelMetas, labelMetas) } } From b7e58f6dcee79c634126f7f9cf60caf42719832c Mon Sep 17 00:00:00 2001 From: DO YUNG YOON Date: Wed, 26 Dec 2018 11:29:50 +0900 Subject: [PATCH 03/13] change S2GraphSource to use DeserializeUtil directly on Result. --- .../apache/s2graph/s2jobs/task/Source.scala | 23 ++- .../s2jobs/wal/utils/DeserializeUtil.scala | 153 ++++++++++++++++++ .../wal/utils/DeserializeUtilTest.scala | 9 ++ 3 files changed, 181 insertions(+), 4 deletions(-) create mode 100644 s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala index db04088a..0abbf4c3 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala @@ -19,12 +19,13 @@ package org.apache.s2graph.s2jobs.task +import org.apache.s2graph.core.types.HBaseType import org.apache.s2graph.core.{JSONParser, Management} import org.apache.s2graph.s2jobs.Schema import org.apache.s2graph.s2jobs.loader.{HFileGenerator, SparkBulkLoaderTransformer} import org.apache.s2graph.s2jobs.serde.reader.S2GraphCellReader import org.apache.s2graph.s2jobs.serde.writer.RowDataFrameWriter -import org.apache.s2graph.s2jobs.wal.utils.DeserializeUtil +import org.apache.s2graph.s2jobs.wal.utils.{DeserializeUtil, SchemaUtil} import org.apache.spark.sql.{DataFrame, SparkSession} import play.api.libs.json.{JsObject, Json} @@ -139,19 +140,24 @@ class S2GraphSource(conf: TaskConf) extends Source(conf) { override def mandatoryOptions: Set[String] = Set( S2_SOURCE_BULKLOAD_HBASE_ROOT_DIR, S2_SOURCE_BULKLOAD_RESTORE_PATH, - S2_SOURCE_BULKLOAD_HBASE_TABLE_NAMES + S2_SOURCE_BULKLOAD_HBASE_TABLE_NAMES, + "labelNames" ) override def toDF(ss: SparkSession): DataFrame = { val mergedConf = TaskConf.parseHBaseConfigs(conf) ++ TaskConf.parseMetaStoreConfigs(conf) ++ TaskConf.parseLocalCacheConfigs(conf) val config = Management.toConfig(mergedConf) + SchemaUtil.init(config) + + val sc = ss.sparkContext val snapshotPath = conf.options(S2_SOURCE_BULKLOAD_HBASE_ROOT_DIR) val restorePath = conf.options(S2_SOURCE_BULKLOAD_RESTORE_PATH) val tableNames = conf.options(S2_SOURCE_BULKLOAD_HBASE_TABLE_NAMES).split(",") val columnFamily = conf.options.getOrElse(S2_SOURCE_BULKLOAD_HBASE_TABLE_CF, "e") val batchSize = conf.options.getOrElse(S2_SOURCE_BULKLOAD_SCAN_BATCH_SIZE, "1000").toInt + val labelNames = conf.options("labelNames").split(",").toSeq val labelMapping = Map.empty[String, String] val buildDegree = @@ -163,8 +169,17 @@ class S2GraphSource(conf: TaskConf) extends Source(conf) { val cells = HFileGenerator.tableSnapshotDump(ss, config, snapshotPath, restorePath, tableNames, columnFamily, batchSize, labelMapping, buildDegree) - val skvs = cells.flatMap { case (_, result) => - result.listCells().asScala.map(DeserializeUtil.cellToSKeyValue) + + val labelSchema = SchemaUtil.buildLabelSchema(labelNames) + val labelSchemaBCast = sc.broadcast(labelSchema) + val tallSchemaVersions = Set(HBaseType.VERSION4) + + val skvs = cells.mapPartitions { iter => + val labelSchema = labelSchemaBCast.value + iter.flatMap { case (_, result) => + DeserializeUtil.indexEdgeResultToWalsV3(result, labelSchema, tallSchemaVersions) + } +// result.listCells().asScala.map(DeserializeUtil.cellToSKeyValue) } ss.createDataFrame(skvs) diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala index f52e96cc..abc847ea 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala @@ -1,6 +1,7 @@ package org.apache.s2graph.s2jobs.wal.utils import org.apache.hadoop.hbase.Cell +import org.apache.hadoop.hbase.client.Result import org.apache.hadoop.hbase.util.Bytes import org.apache.s2graph.core.{GraphUtil, JSONParser, SnapshotEdge} import org.apache.s2graph.core.schema.{Label, LabelIndex, LabelMeta, ServiceColumn} @@ -91,6 +92,158 @@ object DeserializeUtil { cell.getValue, cell.getTimestamp, SKeyValue.Default) } + case class RowV3Parsed(srcVertexId: VertexId, + labelWithDir: LabelWithDirection, + labelIdxSeq: Byte, + isInverted: Boolean, + pos: Int) + + case class QualifierV3Parsed(idxPropsArray: IndexedSeq[(LabelMeta, InnerValLike)], + tgtVertexId: VertexId, + op: Byte) + + def toRowV3Parsed(cell: Cell): RowV3Parsed = { + //TODO: + val row = cell.getRow + var pos = 0 + val (srcVertexId, srcIdLen) = SourceVertexId.fromBytes(row, pos, row.length, HBaseType.DEFAULT_VERSION) + pos += srcIdLen + val labelWithDir = LabelWithDirection(Bytes.toInt(row, pos, 4)) + pos += 4 + val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(row, pos) + pos += 1 + + RowV3Parsed(srcVertexId, labelWithDir, labelIdxSeq, isInverted, pos) + } + + def toQualifierV3Parsed(cell: Cell, + schemaVer: String, + labelId: Int, + labelIdxSeq: Byte, + labelIndexLabelMetas: Map[Int, Map[Byte, Array[LabelMeta]]]): QualifierV3Parsed = { + //TODO: + val qualifier = cell.getQualifier + + val (idxPropsRaw, endAt) = + StorageDeserializable.bytesToProps(qualifier, 0, schemaVer) + + var pos = endAt + + val (tgtVertexIdRaw, tgtVertexIdLen) = + if (endAt == qualifier.length) (HBaseType.defaultTgtVertexId, 0) + else TargetVertexId.fromBytes(qualifier, endAt, qualifier.length, schemaVer) + + pos += tgtVertexIdLen + + val op = + if (qualifier.length == pos) GraphUtil.defaultOpByte + else qualifier.last + + val idxPropsArray = toIndexProps(idxPropsRaw, labelId, labelIdxSeq, labelIndexLabelMetas) + + QualifierV3Parsed(idxPropsArray, tgtVertexIdRaw, op) + } + + def toIndexProps(idxPropsRaw: Array[(LabelMeta, InnerValLike)], + labelId: Int, + labelIdxSeq: Byte, + labelIndexLabelMetas: Map[Int, Map[Byte, Array[LabelMeta]]]): IndexedSeq[(LabelMeta, InnerValLike)] = { + val indexLabelMetas = labelIndexLabelMetas.getOrElse(labelId, throw new IllegalArgumentException(s"${labelId} not found in labelIndexLabelMetas")) + val sortKeyTypesArray = indexLabelMetas.getOrElse(labelIdxSeq, throw new IllegalArgumentException(s"invalid index seq for meta array: ${labelId}, ${labelIdxSeq}")) + + val size = idxPropsRaw.length + (0 until size).map { ith => + val meta = sortKeyTypesArray(ith) + val (k, v) = idxPropsRaw(ith) + meta -> v + } + } + + def toMetaProps(cell: Cell, + labelMetas: Map[Int, Map[Byte, LabelMeta]], + labelId: Int, + op: Byte, + schemaVer: String) = { + val value = cell.getValue + /* process props */ + + if (op == GraphUtil.operations("incrementCount")) { + // val countVal = Bytes.toLong(kv.value) + val countVal = StorageDeserializable.bytesToLong(value, 0) + Array(LabelMeta.count -> InnerVal.withLong(countVal, schemaVer)) + } else { + val (props, endAt) = bytesToKeyValues(value, 0, value.length, schemaVer, labelMetas(labelId)) + props + } + } + + def indexEdgeResultToWalsV3(result: Result, + labelSchema: LabelSchema, + tallSchemaVersions: Set[String], + tgtDirection: Int = 0): Iterable[WalLog] = { + import scala.collection.mutable + val buffer = mutable.ArrayBuffer.empty[WalLog] + val LabelSchema(labelService, labels, _, labelIndexLabelMetas, labelMetas) = labelSchema + + val scanner = result.cellScanner() + + var rowV3Parsed: RowV3Parsed = null + var breakOut = false + + while (scanner.advance() && !breakOut) { + val cell = scanner.current() + + if (rowV3Parsed == null) { + rowV3Parsed = toRowV3Parsed(cell) + } + + val labelWithDir = rowV3Parsed.labelWithDir + val labelId = labelWithDir.labelId + val labelIdxSeq = rowV3Parsed.labelIdxSeq + val srcVertexId = rowV3Parsed.srcVertexId + + breakOut = rowV3Parsed.isInverted || !labels.contains(labelId) || labelWithDir.dir != tgtDirection + + if (!breakOut) { + val label = labels.getOrElse(labelWithDir.labelId, throw new IllegalArgumentException(s"$labelWithDir labelId is not found.")) + val schemaVer = label.schemaVersion + + val qualifierV3Parsed = toQualifierV3Parsed(cell, schemaVer, labelId, labelIdxSeq, labelIndexLabelMetas) + val tgtVertexId = qualifierV3Parsed.tgtVertexId + val idxPropsArray = qualifierV3Parsed.idxPropsArray + val op = qualifierV3Parsed.op + + val metaPropsArray = toMetaProps(cell, labelMetas, labelId, op, schemaVer) + + val mergedProps = (idxPropsArray ++ metaPropsArray).toMap + val tsInnerVal = mergedProps(LabelMeta.timestamp) + val propsJson = for { + (labelMeta, innerValLike) <- mergedProps + jsValue <- JSONParser.innerValToJsValue(innerValLike, labelMeta.dataType) + } yield { + labelMeta.name -> jsValue + } + + val tgtVertexIdInnerId = mergedProps.getOrElse(LabelMeta.to, tgtVertexId.innerId) + + val walLog = WalLog( + tsInnerVal.toIdString().toLong, + GraphUtil.fromOp(op), + "edge", + srcVertexId.innerId.toIdString(), + tgtVertexIdInnerId.toIdString(), + labelService(labelWithDir.labelId), + label.label, + Json.toJson(propsJson).toString() + ) + + buffer += walLog + } + } + + buffer + } + def indexEdgeKeyValueToRow(kv: SKeyValue, cacheElementOpt: Option[SnapshotEdge], labelSchema: LabelSchema, diff --git a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala new file mode 100644 index 00000000..fa7145f5 --- /dev/null +++ b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala @@ -0,0 +1,9 @@ +package org.apache.s2graph.s2jobs.wal.utils + +import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers} + +class DeserializeUtilTest extends FunSuite with Matchers with BeforeAndAfterAll { + test("test indexEdgeResultToWalsV3.") { + + } +} From dfa76a9d0d0d5d932c7a2a9bcfa43c77485adaae Mon Sep 17 00:00:00 2001 From: DO YUNG YOON Date: Wed, 26 Dec 2018 12:56:47 +0900 Subject: [PATCH 04/13] add error handling. --- .../apache/s2graph/s2jobs/task/Source.scala | 2 +- .../s2jobs/wal/utils/DeserializeUtil.scala | 82 ++++++++++-------- .../s2jobs/wal/udfs/HFileParserUDFTest.scala | 84 ++++++++++++------- 3 files changed, 99 insertions(+), 69 deletions(-) diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala index 0abbf4c3..6ec1c9ab 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala @@ -176,10 +176,10 @@ class S2GraphSource(conf: TaskConf) extends Source(conf) { val skvs = cells.mapPartitions { iter => val labelSchema = labelSchemaBCast.value + iter.flatMap { case (_, result) => DeserializeUtil.indexEdgeResultToWalsV3(result, labelSchema, tallSchemaVersions) } -// result.listCells().asScala.map(DeserializeUtil.cellToSKeyValue) } ss.createDataFrame(skvs) diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala index abc847ea..6c169395 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala @@ -1,6 +1,7 @@ package org.apache.s2graph.s2jobs.wal.utils -import org.apache.hadoop.hbase.Cell +import org.apache.hadoop.hbase.KeyValue.Type +import org.apache.hadoop.hbase.{Cell, CellUtil} import org.apache.hadoop.hbase.client.Result import org.apache.hadoop.hbase.util.Bytes import org.apache.s2graph.core.{GraphUtil, JSONParser, SnapshotEdge} @@ -13,6 +14,8 @@ import org.apache.s2graph.s2jobs.wal.{LabelSchema, WalLog} import org.apache.spark.sql.Row import play.api.libs.json.Json +import scala.util.Try + object DeserializeUtil { @@ -92,6 +95,10 @@ object DeserializeUtil { cell.getValue, cell.getTimestamp, SKeyValue.Default) } + def sKeyValueToCell(skv: SKeyValue): Cell = { + CellUtil.createCell(skv.row, skv.cf, skv.qualifier, skv.timestamp, 4.toByte, skv.value) + } + case class RowV3Parsed(srcVertexId: VertexId, labelWithDir: LabelWithDirection, labelIdxSeq: Byte, @@ -116,14 +123,11 @@ object DeserializeUtil { RowV3Parsed(srcVertexId, labelWithDir, labelIdxSeq, isInverted, pos) } - def toQualifierV3Parsed(cell: Cell, + def toQualifierV3Parsed(qualifier: Array[Byte], schemaVer: String, labelId: Int, labelIdxSeq: Byte, - labelIndexLabelMetas: Map[Int, Map[Byte, Array[LabelMeta]]]): QualifierV3Parsed = { - //TODO: - val qualifier = cell.getQualifier - + labelIndexLabelMetas: Map[Int, Map[Byte, Array[LabelMeta]]]): Try[QualifierV3Parsed] = Try { val (idxPropsRaw, endAt) = StorageDeserializable.bytesToProps(qualifier, 0, schemaVer) @@ -179,7 +183,7 @@ object DeserializeUtil { def indexEdgeResultToWalsV3(result: Result, labelSchema: LabelSchema, - tallSchemaVersions: Set[String], + tallSchemaVersions: Set[String] = Set(HBaseType.VERSION4), tgtDirection: Int = 0): Iterable[WalLog] = { import scala.collection.mutable val buffer = mutable.ArrayBuffer.empty[WalLog] @@ -205,39 +209,45 @@ object DeserializeUtil { breakOut = rowV3Parsed.isInverted || !labels.contains(labelId) || labelWithDir.dir != tgtDirection if (!breakOut) { - val label = labels.getOrElse(labelWithDir.labelId, throw new IllegalArgumentException(s"$labelWithDir labelId is not found.")) - val schemaVer = label.schemaVersion + val qualifier = cell.getQualifier + val validV3Qualifier = !qualifier.isEmpty + + if (validV3Qualifier) { + val label = labels(labelId) + val schemaVer = label.schemaVersion + + toQualifierV3Parsed(qualifier, schemaVer, labelId, labelIdxSeq, labelIndexLabelMetas).foreach { qualifierV3Parsed => + val tgtVertexId = qualifierV3Parsed.tgtVertexId + val idxPropsArray = qualifierV3Parsed.idxPropsArray + val op = qualifierV3Parsed.op + + val metaPropsArray = toMetaProps(cell, labelMetas, labelId, op, schemaVer) + + val mergedProps = (idxPropsArray ++ metaPropsArray).toMap + val tsInnerVal = mergedProps(LabelMeta.timestamp) + val propsJson = for { + (labelMeta, innerValLike) <- mergedProps + jsValue <- JSONParser.innerValToJsValue(innerValLike, labelMeta.dataType) + } yield { + labelMeta.name -> jsValue + } - val qualifierV3Parsed = toQualifierV3Parsed(cell, schemaVer, labelId, labelIdxSeq, labelIndexLabelMetas) - val tgtVertexId = qualifierV3Parsed.tgtVertexId - val idxPropsArray = qualifierV3Parsed.idxPropsArray - val op = qualifierV3Parsed.op + val tgtVertexIdInnerId = mergedProps.getOrElse(LabelMeta.to, tgtVertexId.innerId) - val metaPropsArray = toMetaProps(cell, labelMetas, labelId, op, schemaVer) + val walLog = WalLog( + tsInnerVal.toIdString().toLong, + GraphUtil.fromOp(op), + "edge", + srcVertexId.innerId.toIdString(), + tgtVertexIdInnerId.toIdString(), + labelService(labelWithDir.labelId), + label.label, + Json.toJson(propsJson).toString() + ) - val mergedProps = (idxPropsArray ++ metaPropsArray).toMap - val tsInnerVal = mergedProps(LabelMeta.timestamp) - val propsJson = for { - (labelMeta, innerValLike) <- mergedProps - jsValue <- JSONParser.innerValToJsValue(innerValLike, labelMeta.dataType) - } yield { - labelMeta.name -> jsValue + buffer += walLog + } } - - val tgtVertexIdInnerId = mergedProps.getOrElse(LabelMeta.to, tgtVertexId.innerId) - - val walLog = WalLog( - tsInnerVal.toIdString().toLong, - GraphUtil.fromOp(op), - "edge", - srcVertexId.innerId.toIdString(), - tgtVertexIdInnerId.toIdString(), - labelService(labelWithDir.labelId), - label.label, - Json.toJson(propsJson).toString() - ) - - buffer += walLog } } diff --git a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala index a5ed70df..45c65aa5 100644 --- a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala +++ b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala @@ -1,45 +1,65 @@ package org.apache.s2graph.s2jobs.wal.udfs import com.holdenkarau.spark.testing.DataFrameSuiteBase +import com.typesafe.config.ConfigFactory +import org.apache.hadoop.hbase.client.Result +import org.apache.s2graph.core.storage.SKeyValue +import org.apache.s2graph.s2jobs.wal.utils.{DeserializeUtil, SchemaUtil} import org.apache.spark.sql.SaveMode import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers} class HFileParserUDFTest extends FunSuite with Matchers with BeforeAndAfterAll with DataFrameSuiteBase { test("deserialize result udf.") { + import spark.implicits._ - val sKeyValueParser = new HFileParserUDF - sKeyValueParser.register(spark, "to_edge", Map( - "elementType" -> "indexedge", - "labelNames" -> "talk_friend", - "db.default.url" -> "jdbc:mysql://comm-s2graph.mydb.iwilab.com/graph", - "db.default.driver" -> "com.mysql.jdbc.Driver", - "db.default.user" -> "graph", - "db.default.password" -> "graph" - )) - - val df = spark.read.parquet("test.parquet").limit(100) - df.printSchema() +// val sKeyValueParser = new HFileParserUDF +// sKeyValueParser.register(spark, "to_edge", Map( +// "elementType" -> "indexedge", +// "labelNames" -> "talk_friend", +// "db.default.url" -> "jdbc:mysql://comm-s2graph.mydb.iwilab.com/graph", +// "db.default.driver" -> "com.mysql.jdbc.Driver", +// "db.default.user" -> "graph", +// "db.default.password" -> "graph" +// )) + val config = ConfigFactory.load() + SchemaUtil.init(config) + + val labelSchema = SchemaUtil.buildLabelSchema(Seq("talk_friend")) + val labelSchemaBCast = sc.broadcast(labelSchema) + +// val df = spark.read.parquet("test.parquet").limit(100) + val df = spark.read.parquet("test.parquet").as[SKeyValue].mapPartitions { iter => + val labelSchema = labelSchemaBCast.value - df.createOrReplaceTempView("kvs") - -// val sql = """select *, to_edge(struct(*)) as wal from kvs""" - val sql = - """ - |SELECT wal - |FROM ( - | SELECT to_edge(struct(*)) as wal - | FROM kvs - |) - |WHERE wal is not null - """.stripMargin - - val startedAt = System.currentTimeMillis() - val result = spark.sql(sql) - // println(s"[Count]: ${result.count()}") -// result.write.mode(SaveMode.Overwrite).json("./parsed.json") - result.show(false) - val duration = System.currentTimeMillis() - startedAt - println("[Duration]: " + duration) + iter.flatMap { skv => + val result = Result.create(Array(DeserializeUtil.sKeyValueToCell(skv))) + DeserializeUtil.indexEdgeResultToWalsV3(result, labelSchema) + } + } + + df.printSchema() + df.show(1000, false) +// +// df.createOrReplaceTempView("kvs") +// +//// val sql = """select *, to_edge(struct(*)) as wal from kvs""" +// val sql = +// """ +// |SELECT wal +// |FROM ( +// | SELECT to_edge(struct(*)) as wal +// | FROM kvs +// |) +// |WHERE wal is not null +// """.stripMargin +// +// val startedAt = System.currentTimeMillis() +// val result = spark.sql(sql) +// // println(s"[Count]: ${result.count()}") +//// result.write.mode(SaveMode.Overwrite).json("./parsed.json") +// result.show(false) +// val duration = System.currentTimeMillis() - startedAt +// println("[Duration]: " + duration) } } From 33388de1732beaea8375dee8db74a2c4f619603e Mon Sep 17 00:00:00 2001 From: DO YUNG YOON Date: Thu, 27 Dec 2018 13:55:36 +0900 Subject: [PATCH 05/13] directly deserialize cell. --- .../apache/s2graph/s2jobs/task/Source.scala | 32 ++- .../s2jobs/wal/udfs/HFileParserUDF.scala | 43 +++- .../s2jobs/wal/utils/DeserializeUtil.scala | 236 +++++++++++++++++- .../s2jobs/wal/udfs/HFileParserUDFTest.scala | 103 ++++---- 4 files changed, 338 insertions(+), 76 deletions(-) diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala index 6ec1c9ab..65e68f20 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala @@ -19,6 +19,9 @@ package org.apache.s2graph.s2jobs.task +import org.apache.hadoop.hbase.Cell +import org.apache.hadoop.hbase.client.Result +import org.apache.s2graph.core.storage.SKeyValue import org.apache.s2graph.core.types.HBaseType import org.apache.s2graph.core.{JSONParser, Management} import org.apache.s2graph.s2jobs.Schema @@ -26,9 +29,10 @@ import org.apache.s2graph.s2jobs.loader.{HFileGenerator, SparkBulkLoaderTransfor import org.apache.s2graph.s2jobs.serde.reader.S2GraphCellReader import org.apache.s2graph.s2jobs.serde.writer.RowDataFrameWriter import org.apache.s2graph.s2jobs.wal.utils.{DeserializeUtil, SchemaUtil} -import org.apache.spark.sql.{DataFrame, SparkSession} +import org.apache.spark.sql.{DataFrame, Encoders, SparkSession} import play.api.libs.json.{JsObject, Json} +import scala.reflect.runtime.universe._ /** * Source @@ -121,6 +125,7 @@ class FileSource(conf:TaskConf) extends Source(conf) { } } + class HiveSource(conf:TaskConf) extends Source(conf) { override def mandatoryOptions: Set[String] = Set("database", "table") @@ -133,8 +138,24 @@ class HiveSource(conf:TaskConf) extends Source(conf) { } } +case class HResult(table: Array[Byte], + row: Array[Byte], + cf: Array[Byte], + kvs: Seq[PartialKeyValue]) + +object PartialKeyValue { + def apply(cell: Cell): PartialKeyValue = { + PartialKeyValue(cell.getQualifier, cell.getValue, cell.getTimestamp) + } +} +case class PartialKeyValue(qualifier: Array[Byte], + value: Array[Byte], + timestamp: Long) + + class S2GraphSource(conf: TaskConf) extends Source(conf) { import org.apache.s2graph.spark.sql.streaming.S2SourceConfigs._ + import scala.collection.mutable import scala.collection.JavaConverters._ override def mandatoryOptions: Set[String] = Set( @@ -169,19 +190,20 @@ class S2GraphSource(conf: TaskConf) extends Source(conf) { val cells = HFileGenerator.tableSnapshotDump(ss, config, snapshotPath, restorePath, tableNames, columnFamily, batchSize, labelMapping, buildDegree) - val labelSchema = SchemaUtil.buildLabelSchema(labelNames) val labelSchemaBCast = sc.broadcast(labelSchema) val tallSchemaVersions = Set(HBaseType.VERSION4) + val tgtDirection = 0 - val skvs = cells.mapPartitions { iter => + val results = cells.mapPartitions { iter => val labelSchema = labelSchemaBCast.value iter.flatMap { case (_, result) => - DeserializeUtil.indexEdgeResultToWalsV3(result, labelSchema, tallSchemaVersions) + DeserializeUtil.resultToWals(result, labelSchema, tallSchemaVersions, tgtDirection) } + } - ss.createDataFrame(skvs) + ss.createDataFrame(results) } } \ No newline at end of file diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDF.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDF.scala index 119115bd..4ed3a2c7 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDF.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDF.scala @@ -1,12 +1,15 @@ package org.apache.s2graph.s2jobs.wal.udfs +import java.nio.ByteBuffer + import com.typesafe.config.ConfigFactory +import org.apache.hadoop.hbase.client.Result import org.apache.s2graph.core.Management -import org.apache.s2graph.core.schema.{Label, LabelIndex, LabelMeta} import org.apache.s2graph.core.types.HBaseType import org.apache.s2graph.s2jobs.udfs.Udf import org.apache.s2graph.s2jobs.wal.LabelSchema -import org.apache.s2graph.s2jobs.wal.utils.SchemaUtil +import org.apache.s2graph.s2jobs.wal.utils.{DeserializeUtil, SchemaUtil} +import org.apache.spark.serializer.{KryoSerializer, SerializerInstance} import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.functions.udf @@ -14,6 +17,23 @@ import org.apache.spark.sql.functions.udf object HFileParserUDF { import org.apache.s2graph.s2jobs.wal.utils.DeserializeUtil._ + import org.apache.s2graph.core.storage.SKeyValue + +// def resultParser(serDe: SerializerInstance, +// labelSchema: LabelSchema, +// tallSchemaVersion: Set[String]) = { +// udf((row: Array[Byte], resultBytes: Array[Byte]) => { +// val result = serDe.deserialize[Result](ByteBuffer.wrap(resultBytes)) +// DeserializeUtil.resultToWals(row, result, labelSchema, tallSchemaVersion) +// }) +// } + + def walParser(labelSchema: LabelSchema, + tallSchemaVersions: Set[String]) = { + udf((row: Array[Byte], kvsRow: Seq[Row]) => { + DeserializeUtil.indexEdgeResultToWalsV3IterTest(row, kvsRow, labelSchema, tallSchemaVersions) + }) + } def indexEdgeParser(labelSchema: LabelSchema, tallSchemaVersions: Set[String]) = { @@ -42,22 +62,21 @@ class HFileParserUDF extends Udf { val config = ConfigFactory.load(mergedConfig) SchemaUtil.init(config) - val elementType = options.getOrElse("elementType", "indexedge") +// val elementType = options.getOrElse("elementType", "indexedge") val labelNames = options.get("labelNames").map(_.split(",").toSeq).getOrElse(Nil) val labelSchema = SchemaUtil.buildLabelSchema(labelNames) - // logger.error(s"LabelServices: $labelServices") - // logger.error(s"Labels: $labels") - // logger.error(s"LabelIndices: $labelIndices") - // logger.error(s"LabelMetas: $labelMetas") val tallSchemaVersions = Set(HBaseType.VERSION4) - val f = elementType.toLowerCase match { - case "indexedge" => indexEdgeParser(labelSchema, tallSchemaVersions) - case "snapshotedge" => snapshotEdgeParser(labelSchema) - case _ => throw new IllegalArgumentException(s"$elementType is not supported.") - } +// val f = elementType.toLowerCase match { +// case "indexedge" => indexEdgeParser(labelSchema, tallSchemaVersions) +// case "snapshotedge" => snapshotEdgeParser(labelSchema) +// case _ => throw new IllegalArgumentException(s"$elementType is not supported.") +// } + val f = walParser(labelSchema, tallSchemaVersions) +// val serDe = new KryoSerializer(ss.sparkContext.getConf).newInstance() +// val f = resultParser(serDe, labelSchema, tallSchemaVersions) ss.udf.register(name, f) } diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala index 6c169395..9769b6f0 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala @@ -1,22 +1,21 @@ package org.apache.s2graph.s2jobs.wal.utils -import org.apache.hadoop.hbase.KeyValue.Type -import org.apache.hadoop.hbase.{Cell, CellUtil} import org.apache.hadoop.hbase.client.Result import org.apache.hadoop.hbase.util.Bytes -import org.apache.s2graph.core.{GraphUtil, JSONParser, SnapshotEdge} -import org.apache.s2graph.core.schema.{Label, LabelIndex, LabelMeta, ServiceColumn} +import org.apache.hadoop.hbase.{Cell, CellUtil} +import org.apache.s2graph.core.schema.{LabelMeta, ServiceColumn} import org.apache.s2graph.core.storage.SKeyValue import org.apache.s2graph.core.storage.serde.StorageDeserializable import org.apache.s2graph.core.types._ +import org.apache.s2graph.core.{GraphUtil, JSONParser, SnapshotEdge} import org.apache.s2graph.s2jobs.S2GraphHelper.logger import org.apache.s2graph.s2jobs.wal.{LabelSchema, WalLog} import org.apache.spark.sql.Row import play.api.libs.json.Json +import scala.collection.mutable import scala.util.Try - object DeserializeUtil { private def statusCodeWithOp(byte: Byte): (Byte, Byte) = { @@ -109,9 +108,7 @@ object DeserializeUtil { tgtVertexId: VertexId, op: Byte) - def toRowV3Parsed(cell: Cell): RowV3Parsed = { - //TODO: - val row = cell.getRow + def toRowV3Parsed(row: Array[Byte]): RowV3Parsed = { var pos = 0 val (srcVertexId, srcIdLen) = SourceVertexId.fromBytes(row, pos, row.length, HBaseType.DEFAULT_VERSION) pos += srcIdLen @@ -163,12 +160,11 @@ object DeserializeUtil { } } - def toMetaProps(cell: Cell, + def toMetaProps(value: Array[Byte], labelMetas: Map[Int, Map[Byte, LabelMeta]], labelId: Int, op: Byte, schemaVer: String) = { - val value = cell.getValue /* process props */ if (op == GraphUtil.operations("incrementCount")) { @@ -185,7 +181,6 @@ object DeserializeUtil { labelSchema: LabelSchema, tallSchemaVersions: Set[String] = Set(HBaseType.VERSION4), tgtDirection: Int = 0): Iterable[WalLog] = { - import scala.collection.mutable val buffer = mutable.ArrayBuffer.empty[WalLog] val LabelSchema(labelService, labels, _, labelIndexLabelMetas, labelMetas) = labelSchema @@ -198,7 +193,8 @@ object DeserializeUtil { val cell = scanner.current() if (rowV3Parsed == null) { - rowV3Parsed = toRowV3Parsed(cell) + val row = cell.getRow + rowV3Parsed = toRowV3Parsed(row) } val labelWithDir = rowV3Parsed.labelWithDir @@ -220,8 +216,9 @@ object DeserializeUtil { val tgtVertexId = qualifierV3Parsed.tgtVertexId val idxPropsArray = qualifierV3Parsed.idxPropsArray val op = qualifierV3Parsed.op + val value = cell.getValue - val metaPropsArray = toMetaProps(cell, labelMetas, labelId, op, schemaVer) + val metaPropsArray = toMetaProps(value, labelMetas, labelId, op, schemaVer) val mergedProps = (idxPropsArray ++ metaPropsArray).toMap val tsInnerVal = mergedProps(LabelMeta.timestamp) @@ -254,6 +251,219 @@ object DeserializeUtil { buffer } + def deserializeCell(cell: Cell, + rowV3Parsed: RowV3Parsed, + labelSchema: LabelSchema): Option[WalLog] = { + val LabelSchema(labelService, labels, _, labelIndexLabelMetas, labelMetas) = labelSchema + + val labelWithDir = rowV3Parsed.labelWithDir + val labelId = labelWithDir.labelId + val labelIdxSeq = rowV3Parsed.labelIdxSeq + val srcVertexId = rowV3Parsed.srcVertexId + + val qualifier = cell.getQualifier + val validV3Qualifier = !qualifier.isEmpty + + if (!validV3Qualifier) None + else { + val label = labels(labelId) + val schemaVer = label.schemaVersion + + toQualifierV3Parsed(qualifier, schemaVer, labelId, labelIdxSeq, labelIndexLabelMetas).toOption.map { qualifierV3Parsed => + val tgtVertexId = qualifierV3Parsed.tgtVertexId + val idxPropsArray = qualifierV3Parsed.idxPropsArray + val op = qualifierV3Parsed.op + + val value = cell.getValue + + val metaPropsArray = toMetaProps(value, labelMetas, labelId, op, schemaVer) + + val mergedProps = (idxPropsArray ++ metaPropsArray).toMap + val tsInnerVal = mergedProps(LabelMeta.timestamp) + val propsJson = for { + (labelMeta, innerValLike) <- mergedProps + jsValue <- JSONParser.innerValToJsValue(innerValLike, labelMeta.dataType) + } yield { + labelMeta.name -> jsValue + } + + val tgtVertexIdInnerId = mergedProps.getOrElse(LabelMeta.to, tgtVertexId.innerId) + + WalLog( + tsInnerVal.toIdString().toLong, + GraphUtil.fromOp(op), + "edge", + srcVertexId.innerId.toIdString(), + tgtVertexIdInnerId.toIdString(), + labelService(labelWithDir.labelId), + label.label, + Json.toJson(propsJson).toString() + ) + } + } + } + + def resultToWals(result: Result, + labelSchema: LabelSchema, + tallSchemaVersions: Set[String] = Set(HBaseType.VERSION4), + tgtDirection: Int = 0): Seq[WalLog] = { + val rawCells = result.rawCells() + + if (rawCells.isEmpty) Nil + else { + val LabelSchema(labelService, labels, _, labelIndexLabelMetas, labelMetas) = labelSchema + + val head = rawCells.head + val row = head.getRow + val rowV3Parsed = toRowV3Parsed(row) + + val labelWithDir = rowV3Parsed.labelWithDir + val labelId = labelWithDir.labelId + + val inValidRow = rowV3Parsed.isInverted || !labels.contains(labelId) || labelWithDir.dir != tgtDirection + + if (inValidRow) Nil + else { + rawCells.flatMap { cell => + deserializeCell(cell, rowV3Parsed, labelSchema) + } + } + } + } + + def indexEdgeResultToWalsV3IterTest(row: Array[Byte], + kvsRows: Seq[Row], + labelSchema: LabelSchema, + tallSchemaVersions: Set[String] = Set(HBaseType.VERSION4), + tgtDirection: Int = 0): Seq[WalLog] = { + val buffer = mutable.ListBuffer.empty[WalLog] + val LabelSchema(labelService, labels, _, labelIndexLabelMetas, labelMetas) = labelSchema + + + val rowV3Parsed = toRowV3Parsed(row) + + val labelWithDir = rowV3Parsed.labelWithDir + val labelId = labelWithDir.labelId + val labelIdxSeq = rowV3Parsed.labelIdxSeq + val srcVertexId = rowV3Parsed.srcVertexId + + val inValidRow = rowV3Parsed.isInverted || !labels.contains(labelId) || labelWithDir.dir != tgtDirection + + if (inValidRow) Seq.empty[WalLog] + else { + kvsRows.foreach { row => + val qualifier = row.getAs[Array[Byte]]("qualifier") + val validV3Qualifier = !qualifier.isEmpty + if (validV3Qualifier) { + val label = labels(labelId) + val schemaVer = label.schemaVersion + + toQualifierV3Parsed(qualifier, schemaVer, labelId, labelIdxSeq, labelIndexLabelMetas).foreach { qualifierV3Parsed => + val tgtVertexId = qualifierV3Parsed.tgtVertexId + val idxPropsArray = qualifierV3Parsed.idxPropsArray + val op = qualifierV3Parsed.op + + val value = row.getAs[Array[Byte]]("value") + + val metaPropsArray = toMetaProps(value, labelMetas, labelId, op, schemaVer) + + val mergedProps = (idxPropsArray ++ metaPropsArray).toMap + val tsInnerVal = mergedProps(LabelMeta.timestamp) + val propsJson = for { + (labelMeta, innerValLike) <- mergedProps + jsValue <- JSONParser.innerValToJsValue(innerValLike, labelMeta.dataType) + } yield { + labelMeta.name -> jsValue + } + + val tgtVertexIdInnerId = mergedProps.getOrElse(LabelMeta.to, tgtVertexId.innerId) + + val walLog = WalLog( + tsInnerVal.toIdString().toLong, + GraphUtil.fromOp(op), + "edge", + srcVertexId.innerId.toIdString(), + tgtVertexIdInnerId.toIdString(), + labelService(labelWithDir.labelId), + label.label, + Json.toJson(propsJson).toString() + ) + + buffer += walLog + } + } + } + + buffer + } + } + + def indexEdgeResultToWalsV3Iter(kvs: Seq[SKeyValue], + labelSchema: LabelSchema, + tallSchemaVersions: Set[String] = Set(HBaseType.VERSION4), + tgtDirection: Int = 0): Seq[WalLog] = { + val buffer = mutable.ListBuffer.empty[WalLog] + val LabelSchema(labelService, labels, _, labelIndexLabelMetas, labelMetas) = labelSchema + + + val headKV = kvs.head + val rowV3Parsed = toRowV3Parsed(headKV.row) + + val labelWithDir = rowV3Parsed.labelWithDir + val labelId = labelWithDir.labelId + val labelIdxSeq = rowV3Parsed.labelIdxSeq + val srcVertexId = rowV3Parsed.srcVertexId + + val inValidRow = rowV3Parsed.isInverted || !labels.contains(labelId) || labelWithDir.dir != tgtDirection + + if (inValidRow) Seq.empty[WalLog] + else { + kvs.foreach { kv => + val qualifier = kv.qualifier + val validV3Qualifier = !qualifier.isEmpty + if (validV3Qualifier) { + val label = labels(labelId) + val schemaVer = label.schemaVersion + + toQualifierV3Parsed(qualifier, schemaVer, labelId, labelIdxSeq, labelIndexLabelMetas).foreach { qualifierV3Parsed => + val tgtVertexId = qualifierV3Parsed.tgtVertexId + val idxPropsArray = qualifierV3Parsed.idxPropsArray + val op = qualifierV3Parsed.op + + val value = kv.value + val metaPropsArray = toMetaProps(value, labelMetas, labelId, op, schemaVer) + + val mergedProps = (idxPropsArray ++ metaPropsArray).toMap + val tsInnerVal = mergedProps(LabelMeta.timestamp) + val propsJson = for { + (labelMeta, innerValLike) <- mergedProps + jsValue <- JSONParser.innerValToJsValue(innerValLike, labelMeta.dataType) + } yield { + labelMeta.name -> jsValue + } + + val tgtVertexIdInnerId = mergedProps.getOrElse(LabelMeta.to, tgtVertexId.innerId) + + val walLog = WalLog( + tsInnerVal.toIdString().toLong, + GraphUtil.fromOp(op), + "edge", + srcVertexId.innerId.toIdString(), + tgtVertexIdInnerId.toIdString(), + labelService(labelWithDir.labelId), + label.label, + Json.toJson(propsJson).toString() + ) + + buffer += walLog + } + } + } + + buffer + } + } + def indexEdgeKeyValueToRow(kv: SKeyValue, cacheElementOpt: Option[SnapshotEdge], labelSchema: LabelSchema, diff --git a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala index 45c65aa5..ca510003 100644 --- a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala +++ b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala @@ -4,62 +4,73 @@ import com.holdenkarau.spark.testing.DataFrameSuiteBase import com.typesafe.config.ConfigFactory import org.apache.hadoop.hbase.client.Result import org.apache.s2graph.core.storage.SKeyValue +import org.apache.s2graph.s2jobs.task._ import org.apache.s2graph.s2jobs.wal.utils.{DeserializeUtil, SchemaUtil} -import org.apache.spark.sql.SaveMode +import org.apache.spark.sql.{Encoders, Row, SaveMode} import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers} +import scala.reflect.runtime.universe._ + class HFileParserUDFTest extends FunSuite with Matchers with BeforeAndAfterAll with DataFrameSuiteBase { - test("deserialize result udf.") { + test("transform") { import spark.implicits._ +// implicit val tt = typeTag[HResult] +// implicit val enc = Encoders.kryo[HResult] + + val outputPath = "transform.parquet" + val df = + spark.read.parquet("direct.parquet").limit(100) -// val sKeyValueParser = new HFileParserUDF -// sKeyValueParser.register(spark, "to_edge", Map( -// "elementType" -> "indexedge", -// "labelNames" -> "talk_friend", -// "db.default.url" -> "jdbc:mysql://comm-s2graph.mydb.iwilab.com/graph", -// "db.default.driver" -> "com.mysql.jdbc.Driver", -// "db.default.user" -> "graph", -// "db.default.password" -> "graph" -// )) - val config = ConfigFactory.load() - SchemaUtil.init(config) - - val labelSchema = SchemaUtil.buildLabelSchema(Seq("talk_friend")) - val labelSchemaBCast = sc.broadcast(labelSchema) - -// val df = spark.read.parquet("test.parquet").limit(100) - val df = spark.read.parquet("test.parquet").as[SKeyValue].mapPartitions { iter => - val labelSchema = labelSchemaBCast.value - - iter.flatMap { skv => - val result = Result.create(Array(DeserializeUtil.sKeyValueToCell(skv))) - DeserializeUtil.indexEdgeResultToWalsV3(result, labelSchema) + val results = df.map { row => + val kvs = row.getAs[Seq[Row]]("kvs").map { row => + DeserializeUtil.sKeyValueFromRow(row) } + + val table = kvs.head.table + val rowKey = kvs.head.row + val cf = kvs.head.cf + + val partialKVs = kvs.map { kv => PartialKeyValue(kv.qualifier, kv.value, kv.timestamp) } + HResult(table, rowKey, cf, partialKVs) } + results.write.mode(SaveMode.Overwrite).parquet(outputPath) + } + + test("deserialize result udf.") { + import spark.implicits._ + + val sKeyValueParser = new HFileParserUDF + sKeyValueParser.register(spark, "to_wal", Map( + "elementType" -> "indexedge", + "labelNames" -> "talk_friend", + "db.default.url" -> "jdbc:mysql://comm-s2graph.mydb.iwilab.com/graph", + "db.default.driver" -> "com.mysql.jdbc.Driver", + "db.default.user" -> "graph", + "db.default.password" -> "graph" + )) + + val df = + spark.read.parquet("transform.parquet") + df.printSchema() - df.show(1000, false) -// -// df.createOrReplaceTempView("kvs") -// -//// val sql = """select *, to_edge(struct(*)) as wal from kvs""" -// val sql = -// """ -// |SELECT wal -// |FROM ( -// | SELECT to_edge(struct(*)) as wal -// | FROM kvs -// |) -// |WHERE wal is not null -// """.stripMargin -// -// val startedAt = System.currentTimeMillis() -// val result = spark.sql(sql) -// // println(s"[Count]: ${result.count()}") -//// result.write.mode(SaveMode.Overwrite).json("./parsed.json") -// result.show(false) -// val duration = System.currentTimeMillis() - startedAt -// println("[Duration]: " + duration) + df.createOrReplaceTempView("results") + + + val sql = + """ + |SELECT wal.* + |FROM ( + | SELECT explode(to_wal(row, kvs)) as wal + | FROM results + |) + """.stripMargin + + + val result = spark.sql(sql) + result.printSchema() + + result.write.mode(SaveMode.Overwrite).json("./parsed.json") } } From 2942e42eb00e5f9fa19ecf23319d615df3a2e87a Mon Sep 17 00:00:00 2001 From: DO YUNG YOON Date: Sat, 29 Dec 2018 09:47:41 +0900 Subject: [PATCH 06/13] tmp. --- .../apache/s2graph/core/types/VertexId.scala | 12 + .../apache/s2graph/s2jobs/task/Source.scala | 2 +- .../s2graph/s2jobs/wal/ColumnSchema.scala | 26 + .../apache/s2graph/s2jobs/wal/WalLog.scala | 10 +- .../s2jobs/wal/udfs/HFileParserUDF.scala | 164 +++--- .../s2jobs/wal/utils/DeserializeUtil.scala | 539 ++++++------------ .../s2jobs/wal/utils/SerializeUtil.scala | 38 ++ .../s2jobs/wal/udfs/HFileParserUDFTest.scala | 2 + .../wal/utils/DeserializeUtilTest.scala | 2 +- 9 files changed, 339 insertions(+), 456 deletions(-) create mode 100644 s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/ColumnSchema.scala create mode 100644 s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala diff --git a/s2core/src/main/scala/org/apache/s2graph/core/types/VertexId.scala b/s2core/src/main/scala/org/apache/s2graph/core/types/VertexId.scala index 1ad56b36..5cf9f9fd 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/types/VertexId.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/types/VertexId.scala @@ -29,6 +29,18 @@ import play.api.libs.json.Json object VertexId extends HBaseDeserializable { import HBaseType._ + def fromBytesRaw(bytes: Array[Byte], + offset: Int, + len: Int, + version: String = DEFAULT_VERSION): (InnerValLike, Int, Int) = { + var pos = offset + GraphUtil.bytesForMurMurHash + + val (innerId, numOfBytesUsed) = InnerVal.fromBytes(bytes, pos, len, version, isVertexId = true) + pos += numOfBytesUsed + val colId = Bytes.toInt(bytes, pos, 4) + + (innerId, colId, GraphUtil.bytesForMurMurHash + numOfBytesUsed + 4) + } def fromBytes(bytes: Array[Byte], offset: Int, diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala index 65e68f20..55a2a379 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala @@ -199,7 +199,7 @@ class S2GraphSource(conf: TaskConf) extends Source(conf) { val labelSchema = labelSchemaBCast.value iter.flatMap { case (_, result) => - DeserializeUtil.resultToWals(result, labelSchema, tallSchemaVersions, tgtDirection) + DeserializeUtil.indexEdgeResultToWals(result, labelSchema, tallSchemaVersions, tgtDirection) } } diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/ColumnSchema.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/ColumnSchema.scala new file mode 100644 index 00000000..9cdc1346 --- /dev/null +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/ColumnSchema.scala @@ -0,0 +1,26 @@ +package org.apache.s2graph.s2jobs.wal + +import org.apache.s2graph.core.schema._ + +case class ColumnSchema(columnService: Map[Int, Service], + columns: Map[Int, ServiceColumn], + columnMetas: Map[Int, Map[Int, ColumnMeta]]) { + + val services: Set[Service] = columnService.map(_._2).toSet + val serviceColumns: Map[String, Set[ServiceColumn]] = columnService.groupBy(_._2).map { case (service, ls) => + val columnIds = ls.map(_._1) + + service.serviceName -> columnIds.map(columns).toSet + } + val serviceColumnMetas: Map[(String, String), Map[String, ColumnMeta]] = columnMetas.map { case (columId, m) => + val service = columnService(columId) + val columnName = columns(columId).columnName + + val rev = m.map { case (seq, columnMeta) => + columnMeta.name -> columnMeta + } + + (service.serviceName, columnName) -> rev + } + +} diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/WalLog.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/WalLog.scala index ad696a9c..3c23721e 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/WalLog.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/WalLog.scala @@ -176,6 +176,14 @@ case class WalLog(timestamp: Long, } } +case class WalVertex(timestamp: Long, + operation: String, + elem: String, + id: String, + service: String, + column: String, + props: String) + object WalLog { val orderByTsAsc = Ordering.by[WalLog, Long](walLog => walLog.timestamp) @@ -203,6 +211,4 @@ object WalLog { WalLog(timestamp, operation, elem, from, to, service, label, props) } - - } diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDF.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDF.scala index 4ed3a2c7..7a73f191 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDF.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDF.scala @@ -1,83 +1,83 @@ -package org.apache.s2graph.s2jobs.wal.udfs - -import java.nio.ByteBuffer - -import com.typesafe.config.ConfigFactory -import org.apache.hadoop.hbase.client.Result -import org.apache.s2graph.core.Management -import org.apache.s2graph.core.types.HBaseType -import org.apache.s2graph.s2jobs.udfs.Udf -import org.apache.s2graph.s2jobs.wal.LabelSchema -import org.apache.s2graph.s2jobs.wal.utils.{DeserializeUtil, SchemaUtil} -import org.apache.spark.serializer.{KryoSerializer, SerializerInstance} -import org.apache.spark.sql.{Row, SparkSession} -import org.apache.spark.sql.functions.udf - - -object HFileParserUDF { - - import org.apache.s2graph.s2jobs.wal.utils.DeserializeUtil._ - import org.apache.s2graph.core.storage.SKeyValue - -// def resultParser(serDe: SerializerInstance, -// labelSchema: LabelSchema, -// tallSchemaVersion: Set[String]) = { -// udf((row: Array[Byte], resultBytes: Array[Byte]) => { -// val result = serDe.deserialize[Result](ByteBuffer.wrap(resultBytes)) -// DeserializeUtil.resultToWals(row, result, labelSchema, tallSchemaVersion) -// }) +//package org.apache.s2graph.s2jobs.wal.udfs +// +//import java.nio.ByteBuffer +// +//import com.typesafe.config.ConfigFactory +//import org.apache.hadoop.hbase.client.Result +//import org.apache.s2graph.core.Management +//import org.apache.s2graph.core.types.HBaseType +//import org.apache.s2graph.s2jobs.udfs.Udf +//import org.apache.s2graph.s2jobs.wal.LabelSchema +//import org.apache.s2graph.s2jobs.wal.utils.{DeserializeUtil, SchemaUtil} +//import org.apache.spark.serializer.{KryoSerializer, SerializerInstance} +//import org.apache.spark.sql.{Row, SparkSession} +//import org.apache.spark.sql.functions.udf +// +// +//object HFileParserUDF { +// +// import org.apache.s2graph.s2jobs.wal.utils.DeserializeUtil._ +// import org.apache.s2graph.core.storage.SKeyValue +// +//// def resultParser(serDe: SerializerInstance, +//// labelSchema: LabelSchema, +//// tallSchemaVersion: Set[String]) = { +//// udf((row: Array[Byte], resultBytes: Array[Byte]) => { +//// val result = serDe.deserialize[Result](ByteBuffer.wrap(resultBytes)) +//// DeserializeUtil.resultToWals(row, result, labelSchema, tallSchemaVersion) +//// }) +//// } +// +//// def walParser(labelSchema: LabelSchema, +//// tallSchemaVersions: Set[String]) = { +//// udf((row: Array[Byte], kvsRow: Seq[Row]) => { +//// DeserializeUtil.indexEdgeResultToWalsV3IterTest(row, kvsRow, labelSchema, tallSchemaVersions) +//// }) +//// } +// +//// def indexEdgeParser(labelSchema: LabelSchema, +//// tallSchemaVersions: Set[String]) = { +//// udf((row: Row) => { +//// val skv = sKeyValueFromRow(row) +//// +//// indexEdgeKeyValueToRow(skv, None, labelSchema, tallSchemaVersions) +//// }) +//// } +//// +//// def snapshotEdgeParser(labelSchema: LabelSchema) = { +//// udf((row: Row) => { +//// val skv = sKeyValueFromRow(row) +//// snapshotEdgeKeyValueToRow(skv, None, labelSchema) +//// }) +//// } +//} +// +//class HFileParserUDF extends Udf { +// +// import HFileParserUDF._ +// import org.apache.s2graph.s2jobs.wal.utils.TaskConfUtil._ +// +// override def register(ss: SparkSession, name: String, options: Map[String, String]): Unit = { +// val mergedConfig = Management.toConfig(parseMetaStoreConfigs(options)) +// val config = ConfigFactory.load(mergedConfig) +// SchemaUtil.init(config) +// +//// val elementType = options.getOrElse("elementType", "indexedge") +// val labelNames = options.get("labelNames").map(_.split(",").toSeq).getOrElse(Nil) +// +// val labelSchema = SchemaUtil.buildLabelSchema(labelNames) +// +// val tallSchemaVersions = Set(HBaseType.VERSION4) +// +//// val f = elementType.toLowerCase match { +//// case "indexedge" => indexEdgeParser(labelSchema, tallSchemaVersions) +//// case "snapshotedge" => snapshotEdgeParser(labelSchema) +//// case _ => throw new IllegalArgumentException(s"$elementType is not supported.") +//// } +// val f = walParser(labelSchema, tallSchemaVersions) +//// val serDe = new KryoSerializer(ss.sparkContext.getConf).newInstance() +//// val f = resultParser(serDe, labelSchema, tallSchemaVersions) +// +// ss.udf.register(name, f) // } - - def walParser(labelSchema: LabelSchema, - tallSchemaVersions: Set[String]) = { - udf((row: Array[Byte], kvsRow: Seq[Row]) => { - DeserializeUtil.indexEdgeResultToWalsV3IterTest(row, kvsRow, labelSchema, tallSchemaVersions) - }) - } - - def indexEdgeParser(labelSchema: LabelSchema, - tallSchemaVersions: Set[String]) = { - udf((row: Row) => { - val skv = sKeyValueFromRow(row) - - indexEdgeKeyValueToRow(skv, None, labelSchema, tallSchemaVersions) - }) - } - - def snapshotEdgeParser(labelSchema: LabelSchema) = { - udf((row: Row) => { - val skv = sKeyValueFromRow(row) - snapshotEdgeKeyValueToRow(skv, None, labelSchema) - }) - } -} - -class HFileParserUDF extends Udf { - - import HFileParserUDF._ - import org.apache.s2graph.s2jobs.wal.utils.TaskConfUtil._ - - override def register(ss: SparkSession, name: String, options: Map[String, String]): Unit = { - val mergedConfig = Management.toConfig(parseMetaStoreConfigs(options)) - val config = ConfigFactory.load(mergedConfig) - SchemaUtil.init(config) - -// val elementType = options.getOrElse("elementType", "indexedge") - val labelNames = options.get("labelNames").map(_.split(",").toSeq).getOrElse(Nil) - - val labelSchema = SchemaUtil.buildLabelSchema(labelNames) - - val tallSchemaVersions = Set(HBaseType.VERSION4) - -// val f = elementType.toLowerCase match { -// case "indexedge" => indexEdgeParser(labelSchema, tallSchemaVersions) -// case "snapshotedge" => snapshotEdgeParser(labelSchema) -// case _ => throw new IllegalArgumentException(s"$elementType is not supported.") -// } - val f = walParser(labelSchema, tallSchemaVersions) -// val serDe = new KryoSerializer(ss.sparkContext.getConf).newInstance() -// val f = resultParser(serDe, labelSchema, tallSchemaVersions) - - ss.udf.register(name, f) - } -} \ No newline at end of file +//} \ No newline at end of file diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala index 9769b6f0..4154a80d 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala @@ -3,18 +3,18 @@ package org.apache.s2graph.s2jobs.wal.utils import org.apache.hadoop.hbase.client.Result import org.apache.hadoop.hbase.util.Bytes import org.apache.hadoop.hbase.{Cell, CellUtil} -import org.apache.s2graph.core.schema.{LabelMeta, ServiceColumn} -import org.apache.s2graph.core.storage.SKeyValue +import org.apache.s2graph.core.schema.{ColumnMeta, LabelMeta, ServiceColumn} +import org.apache.s2graph.core.storage.{CanSKeyValue, SKeyValue} import org.apache.s2graph.core.storage.serde.StorageDeserializable +import org.apache.s2graph.core.storage.serde.StorageDeserializable.bytesToInt import org.apache.s2graph.core.types._ -import org.apache.s2graph.core.{GraphUtil, JSONParser, SnapshotEdge} -import org.apache.s2graph.s2jobs.S2GraphHelper.logger -import org.apache.s2graph.s2jobs.wal.{LabelSchema, WalLog} +import org.apache.s2graph.core.{GraphUtil, JSONParser, S2Vertex} +import org.apache.s2graph.s2jobs.wal.{ColumnSchema, LabelSchema, WalLog, WalVertex} import org.apache.spark.sql.Row import play.api.libs.json.Json -import scala.collection.mutable -import scala.util.Try +import scala.collection.mutable.ListBuffer +import scala.util.{Failure, Success, Try} object DeserializeUtil { @@ -108,6 +108,12 @@ object DeserializeUtil { tgtVertexId: VertexId, op: Byte) + case class SnapshotRowV3Parsed(srcVertexId: VertexId, + tgtVertexId: VertexId, + labelWithDir: LabelWithDirection, + labelIdxSeq: Byte, + isInverted: Boolean) + def toRowV3Parsed(row: Array[Byte]): RowV3Parsed = { var pos = 0 val (srcVertexId, srcIdLen) = SourceVertexId.fromBytes(row, pos, row.length, HBaseType.DEFAULT_VERSION) @@ -120,6 +126,26 @@ object DeserializeUtil { RowV3Parsed(srcVertexId, labelWithDir, labelIdxSeq, isInverted, pos) } + def toQualifierV4Parsed(row: Array[Byte], + offset: Int, + schemaVer: String, + labelId: Int, + labelIdxSeq: Byte, + labelIndexLabelMetas: Map[Int, Map[Byte, Array[LabelMeta]]]): Try[QualifierV3Parsed] = Try { + val (idxPropsRaw, endAt) = StorageDeserializable.bytesToProps(row, offset, schemaVer) + val pos = endAt + + val (tgtVertexIdRaw, tgtVertexIdLen) = + if (endAt == row.length - 1) (HBaseType.defaultTgtVertexId, 0) + else TargetVertexId.fromBytes(row, endAt, row.length - 1, schemaVer) + + val op = row.last + + val idxPropsArray = toIndexProps(idxPropsRaw, labelId, labelIdxSeq, labelIndexLabelMetas) + + QualifierV3Parsed(idxPropsArray, tgtVertexIdRaw, op) + } + def toQualifierV3Parsed(qualifier: Array[Byte], schemaVer: String, labelId: Int, @@ -177,83 +203,36 @@ object DeserializeUtil { } } - def indexEdgeResultToWalsV3(result: Result, - labelSchema: LabelSchema, - tallSchemaVersions: Set[String] = Set(HBaseType.VERSION4), - tgtDirection: Int = 0): Iterable[WalLog] = { - val buffer = mutable.ArrayBuffer.empty[WalLog] - val LabelSchema(labelService, labels, _, labelIndexLabelMetas, labelMetas) = labelSchema + def toSnapshotRowV3Parsed(row: Array[Byte]): Try[SnapshotRowV3Parsed] = Try { + val labelWithDirByteLen = 4 + val labelIndexSeqWithIsInvertedByteLen = 1 - val scanner = result.cellScanner() + val (srcVertexId, srcIdLen) = + SourceVertexId.fromBytes(row, 0, row.length, HBaseType.DEFAULT_VERSION) - var rowV3Parsed: RowV3Parsed = null - var breakOut = false + val isTallSchema = + srcIdLen + labelWithDirByteLen + labelIndexSeqWithIsInvertedByteLen != row.length - while (scanner.advance() && !breakOut) { - val cell = scanner.current() + val (tgtVertexId, pos) = + if (isTallSchema) { + val (tgtId, tgtBytesLen) = InnerVal.fromBytes(row, srcIdLen, row.length, HBaseType.DEFAULT_VERSION) - if (rowV3Parsed == null) { - val row = cell.getRow - rowV3Parsed = toRowV3Parsed(row) + (TargetVertexId(ServiceColumn.Default, tgtId), srcIdLen + tgtBytesLen) + } else { + (TargetVertexId(ServiceColumn.Default, srcVertexId.innerId), srcIdLen) } - val labelWithDir = rowV3Parsed.labelWithDir - val labelId = labelWithDir.labelId - val labelIdxSeq = rowV3Parsed.labelIdxSeq - val srcVertexId = rowV3Parsed.srcVertexId - - breakOut = rowV3Parsed.isInverted || !labels.contains(labelId) || labelWithDir.dir != tgtDirection - - if (!breakOut) { - val qualifier = cell.getQualifier - val validV3Qualifier = !qualifier.isEmpty - - if (validV3Qualifier) { - val label = labels(labelId) - val schemaVer = label.schemaVersion - - toQualifierV3Parsed(qualifier, schemaVer, labelId, labelIdxSeq, labelIndexLabelMetas).foreach { qualifierV3Parsed => - val tgtVertexId = qualifierV3Parsed.tgtVertexId - val idxPropsArray = qualifierV3Parsed.idxPropsArray - val op = qualifierV3Parsed.op - val value = cell.getValue - - val metaPropsArray = toMetaProps(value, labelMetas, labelId, op, schemaVer) - - val mergedProps = (idxPropsArray ++ metaPropsArray).toMap - val tsInnerVal = mergedProps(LabelMeta.timestamp) - val propsJson = for { - (labelMeta, innerValLike) <- mergedProps - jsValue <- JSONParser.innerValToJsValue(innerValLike, labelMeta.dataType) - } yield { - labelMeta.name -> jsValue - } - - val tgtVertexIdInnerId = mergedProps.getOrElse(LabelMeta.to, tgtVertexId.innerId) - - val walLog = WalLog( - tsInnerVal.toIdString().toLong, - GraphUtil.fromOp(op), - "edge", - srcVertexId.innerId.toIdString(), - tgtVertexIdInnerId.toIdString(), - labelService(labelWithDir.labelId), - label.label, - Json.toJson(propsJson).toString() - ) - - buffer += walLog - } - } - } - } + val labelWithDir = LabelWithDirection(Bytes.toInt(row, pos, labelWithDirByteLen)) + val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(row, pos + labelWithDirByteLen) - buffer + SnapshotRowV3Parsed(srcVertexId, tgtVertexId, labelWithDir, labelIdxSeq, isInverted) } - def deserializeCell(cell: Cell, - rowV3Parsed: RowV3Parsed, - labelSchema: LabelSchema): Option[WalLog] = { + def deserializeIndexEdgeCell(cell: Cell, + row: Array[Byte], + rowV3Parsed: RowV3Parsed, + labelSchema: LabelSchema, + tallSchemaVersions: Set[String]): Option[WalLog] = { val LabelSchema(labelService, labels, _, labelIndexLabelMetas, labelMetas) = labelSchema val labelWithDir = rowV3Parsed.labelWithDir @@ -261,18 +240,26 @@ object DeserializeUtil { val labelIdxSeq = rowV3Parsed.labelIdxSeq val srcVertexId = rowV3Parsed.srcVertexId + val label = labels(labelId) + val schemaVer = label.schemaVersion + val isTallSchema = tallSchemaVersions(label.schemaVersion) + val qualifier = cell.getQualifier + + val isDegree = if (isTallSchema) rowV3Parsed.pos == row.length else qualifier.isEmpty val validV3Qualifier = !qualifier.isEmpty - if (!validV3Qualifier) None + if (isDegree) None + else if (!isTallSchema && !validV3Qualifier) None else { - val label = labels(labelId) - val schemaVer = label.schemaVersion + val qualifierParsedTry = + if (isTallSchema) toQualifierV4Parsed(row, rowV3Parsed.pos, schemaVer, labelId, labelIdxSeq, labelIndexLabelMetas) + else toQualifierV3Parsed(qualifier, schemaVer, labelId, labelIdxSeq, labelIndexLabelMetas) - toQualifierV3Parsed(qualifier, schemaVer, labelId, labelIdxSeq, labelIndexLabelMetas).toOption.map { qualifierV3Parsed => - val tgtVertexId = qualifierV3Parsed.tgtVertexId - val idxPropsArray = qualifierV3Parsed.idxPropsArray - val op = qualifierV3Parsed.op + qualifierParsedTry.toOption.map { qualifierParsed => + val tgtVertexId = qualifierParsed.tgtVertexId + val idxPropsArray = qualifierParsed.idxPropsArray + val op = qualifierParsed.op val value = cell.getValue @@ -303,15 +290,15 @@ object DeserializeUtil { } } - def resultToWals(result: Result, - labelSchema: LabelSchema, - tallSchemaVersions: Set[String] = Set(HBaseType.VERSION4), - tgtDirection: Int = 0): Seq[WalLog] = { + def indexEdgeResultToWals(result: Result, + labelSchema: LabelSchema, + tallSchemaVersions: Set[String], + tgtDirection: Int = 0): Seq[WalLog] = { val rawCells = result.rawCells() if (rawCells.isEmpty) Nil else { - val LabelSchema(labelService, labels, _, labelIndexLabelMetas, labelMetas) = labelSchema + val LabelSchema(_, labels, _, _, _) = labelSchema val head = rawCells.head val row = head.getRow @@ -325,321 +312,133 @@ object DeserializeUtil { if (inValidRow) Nil else { rawCells.flatMap { cell => - deserializeCell(cell, rowV3Parsed, labelSchema) + deserializeIndexEdgeCell(cell, row, rowV3Parsed, labelSchema, tallSchemaVersions) } } } } - def indexEdgeResultToWalsV3IterTest(row: Array[Byte], - kvsRows: Seq[Row], - labelSchema: LabelSchema, - tallSchemaVersions: Set[String] = Set(HBaseType.VERSION4), - tgtDirection: Int = 0): Seq[WalLog] = { - val buffer = mutable.ListBuffer.empty[WalLog] - val LabelSchema(labelService, labels, _, labelIndexLabelMetas, labelMetas) = labelSchema - - - val rowV3Parsed = toRowV3Parsed(row) - - val labelWithDir = rowV3Parsed.labelWithDir - val labelId = labelWithDir.labelId - val labelIdxSeq = rowV3Parsed.labelIdxSeq - val srcVertexId = rowV3Parsed.srcVertexId - val inValidRow = rowV3Parsed.isInverted || !labels.contains(labelId) || labelWithDir.dir != tgtDirection + def snapshotEdgeResultToWals(result: Result, + labelSchema: LabelSchema, + tallSchemaVersions: Set[String]): Seq[WalLog] = { + val rawCells = result.rawCells() - if (inValidRow) Seq.empty[WalLog] + if (rawCells.isEmpty) Nil else { - kvsRows.foreach { row => - val qualifier = row.getAs[Array[Byte]]("qualifier") - val validV3Qualifier = !qualifier.isEmpty - if (validV3Qualifier) { - val label = labels(labelId) - val schemaVer = label.schemaVersion - - toQualifierV3Parsed(qualifier, schemaVer, labelId, labelIdxSeq, labelIndexLabelMetas).foreach { qualifierV3Parsed => - val tgtVertexId = qualifierV3Parsed.tgtVertexId - val idxPropsArray = qualifierV3Parsed.idxPropsArray - val op = qualifierV3Parsed.op - - val value = row.getAs[Array[Byte]]("value") - - val metaPropsArray = toMetaProps(value, labelMetas, labelId, op, schemaVer) - - val mergedProps = (idxPropsArray ++ metaPropsArray).toMap - val tsInnerVal = mergedProps(LabelMeta.timestamp) - val propsJson = for { - (labelMeta, innerValLike) <- mergedProps - jsValue <- JSONParser.innerValToJsValue(innerValLike, labelMeta.dataType) - } yield { - labelMeta.name -> jsValue - } - - val tgtVertexIdInnerId = mergedProps.getOrElse(LabelMeta.to, tgtVertexId.innerId) - - val walLog = WalLog( - tsInnerVal.toIdString().toLong, - GraphUtil.fromOp(op), - "edge", - srcVertexId.innerId.toIdString(), - tgtVertexIdInnerId.toIdString(), - labelService(labelWithDir.labelId), - label.label, - Json.toJson(propsJson).toString() - ) + val LabelSchema(labelService, labels, _, _, labelMetas) = labelSchema + val head = rawCells.head + val row = head.getRow - buffer += walLog + toSnapshotRowV3Parsed(row) match { + case Success(v) => + val SnapshotRowV3Parsed(srcVertexId, tgtVertexId, labelWithDir, _, isInverted) = v + + if (!isInverted) Nil + else { + val label = labels(labelWithDir.labelId) + val schemaVer = label.schemaVersion + + rawCells.map { cell => + val value = cell.getValue + val (_, op) = statusCodeWithOp(value.head) + + val (props, _) = bytesToKeyValuesWithTs(value, 1, schemaVer, labelMetas(labelWithDir.labelId)) + val kvsMap = props.toMap + val tsInnerVal = kvsMap(LabelMeta.timestamp).innerVal + + val propsJson = for { + (labelMeta, innerValLikeWithTs) <- props + jsValue <- JSONParser.innerValToJsValue(innerValLikeWithTs.innerVal, labelMeta.dataType) + } yield { + labelMeta.name -> jsValue + } + + WalLog( + tsInnerVal.toIdString().toLong, + GraphUtil.fromOp(op), + "edge", + srcVertexId.innerId.toIdString(), + tgtVertexId.innerId.toIdString(), + labelService(labelWithDir.labelId), + label.label, + Json.toJson(propsJson.toMap).toString() + ) + } } - } - } - buffer + case Failure(ex) => Nil + } } } - def indexEdgeResultToWalsV3Iter(kvs: Seq[SKeyValue], - labelSchema: LabelSchema, - tallSchemaVersions: Set[String] = Set(HBaseType.VERSION4), - tgtDirection: Int = 0): Seq[WalLog] = { - val buffer = mutable.ListBuffer.empty[WalLog] - val LabelSchema(labelService, labels, _, labelIndexLabelMetas, labelMetas) = labelSchema - + def vertexResultToWals(result: Result, + columnSchema: ColumnSchema, + bytesToInt: (Array[Byte], Int) => Int = bytesToInt): Seq[WalVertex] = { + import scala.collection.mutable - val headKV = kvs.head - val rowV3Parsed = toRowV3Parsed(headKV.row) + val ColumnSchema(columnService, columns, columnMetas) = columnSchema - val labelWithDir = rowV3Parsed.labelWithDir - val labelId = labelWithDir.labelId - val labelIdxSeq = rowV3Parsed.labelIdxSeq - val srcVertexId = rowV3Parsed.srcVertexId - - val inValidRow = rowV3Parsed.isInverted || !labels.contains(labelId) || labelWithDir.dir != tgtDirection + val rawCells = result.rawCells() - if (inValidRow) Seq.empty[WalLog] + if (rawCells.isEmpty) Nil else { - kvs.foreach { kv => - val qualifier = kv.qualifier - val validV3Qualifier = !qualifier.isEmpty - if (validV3Qualifier) { - val label = labels(labelId) - val schemaVer = label.schemaVersion - - toQualifierV3Parsed(qualifier, schemaVer, labelId, labelIdxSeq, labelIndexLabelMetas).foreach { qualifierV3Parsed => - val tgtVertexId = qualifierV3Parsed.tgtVertexId - val idxPropsArray = qualifierV3Parsed.idxPropsArray - val op = qualifierV3Parsed.op - - val value = kv.value - val metaPropsArray = toMetaProps(value, labelMetas, labelId, op, schemaVer) - - val mergedProps = (idxPropsArray ++ metaPropsArray).toMap - val tsInnerVal = mergedProps(LabelMeta.timestamp) - val propsJson = for { - (labelMeta, innerValLike) <- mergedProps - jsValue <- JSONParser.innerValToJsValue(innerValLike, labelMeta.dataType) - } yield { - labelMeta.name -> jsValue - } - - val tgtVertexIdInnerId = mergedProps.getOrElse(LabelMeta.to, tgtVertexId.innerId) + val head = rawCells.head + val row = head.getRow - val walLog = WalLog( - tsInnerVal.toIdString().toLong, - GraphUtil.fromOp(op), - "edge", - srcVertexId.innerId.toIdString(), - tgtVertexIdInnerId.toIdString(), - labelService(labelWithDir.labelId), - label.label, - Json.toJson(propsJson).toString() - ) + val version = HBaseType.DEFAULT_VERSION + val (vertexInnerId, colId, _) = VertexId.fromBytesRaw(row, 0, row.length, version) - buffer += walLog - } - } - } + var maxTs = Long.MinValue + val propsMap = mutable.Map.empty[ColumnMeta, InnerValLike] + val belongLabelIds = mutable.ListBuffer.empty[Int] - buffer - } - } + rawCells.map { cell => + val qualifier = cell.getQualifier - def indexEdgeKeyValueToRow(kv: SKeyValue, - cacheElementOpt: Option[SnapshotEdge], - labelSchema: LabelSchema, - tallSchemaVersions: Set[String]): Option[WalLog] = { - try { - val LabelSchema(labelService, labels, labelIndices, labelIndexLabelMetas, labelMetas) = labelSchema - - var pos = 0 - val (srcVertexId, srcIdLen) = SourceVertexId.fromBytes(kv.row, pos, kv.row.length, HBaseType.DEFAULT_VERSION) - pos += srcIdLen - val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4)) - pos += 4 - val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(kv.row, pos) - pos += 1 + val propKey = + if (qualifier.length == 1) qualifier.head.toInt + else bytesToInt(qualifier, 0) - if (isInverted) None - else if (!labels.contains(labelWithDir.labelId)) None - else { - val label = labels.getOrElse(labelWithDir.labelId, throw new IllegalArgumentException(s"$labelWithDir labelId is not found.")) - val schemaVer = label.schemaVersion + val ts = cell.getTimestamp - val isTallSchema = tallSchemaVersions(label.schemaVersion) - val isDegree = if (isTallSchema) pos == kv.row.length else kv.qualifier.isEmpty + if (ts > maxTs) maxTs = ts - if (isDegree) { - None + if (S2Vertex.isLabelId(propKey)) { + belongLabelIds += S2Vertex.toLabelId(propKey) } else { - // not degree edge - val (idxPropsRaw, endAt) = - if (isTallSchema) StorageDeserializable.bytesToProps(kv.row, pos, schemaVer) - else { - StorageDeserializable.bytesToProps(kv.qualifier, 0, schemaVer) - } - pos = endAt - - val (tgtVertexIdRaw, tgtVertexIdLen) = if (isTallSchema) { - if (endAt == kv.row.length - 1) { - (HBaseType.defaultTgtVertexId, 0) - } else { - TargetVertexId.fromBytes(kv.row, endAt, kv.row.length - 1, schemaVer) - } - } else { - if (endAt == kv.qualifier.length) { - (HBaseType.defaultTgtVertexId, 0) - } else { - TargetVertexId.fromBytes(kv.qualifier, endAt, kv.qualifier.length, schemaVer) - } - } - pos += tgtVertexIdLen - - val op = - if (isTallSchema) kv.row(kv.row.length - 1) - else { - if (kv.qualifier.length == pos) GraphUtil.defaultOpByte - else kv.qualifier(kv.qualifier.length - 1) - } - - val indexLabelMetas = labelIndexLabelMetas.getOrElse(labelWithDir.labelId, throw new IllegalArgumentException(s"${labelWithDir.labelId} not found in labelIndexLabelMetas")) - val sortKeyTypesArray = indexLabelMetas.getOrElse(labelIdxSeq, throw new IllegalArgumentException(s"invalid index seq for meta array: ${label.id.get}, ${labelIdxSeq}")) - - /* process indexProps */ - val size = idxPropsRaw.length - val idxProps = (0 until size).map { ith => - val meta = sortKeyTypesArray(ith) - val (k, v) = idxPropsRaw(ith) - meta -> v - } + val v = cell.getValue - /* process props */ - val metaProps = - if (op == GraphUtil.operations("incrementCount")) { - // val countVal = Bytes.toLong(kv.value) - val countVal = StorageDeserializable.bytesToLong(kv.value, 0) - Array(LabelMeta.count -> InnerVal.withLong(countVal, schemaVer)) - } else { - val (props, endAt) = bytesToKeyValues(kv.value, 0, kv.value.length, schemaVer, labelMetas(labelWithDir.labelId)) - props - } - - val mergedProps = (idxProps ++ metaProps).toMap - val tsInnerVal = mergedProps(LabelMeta.timestamp) - val propsJson = for { - (labelMeta, innerValLike) <- mergedProps - jsValue <- JSONParser.innerValToJsValue(innerValLike, labelMeta.dataType) - } yield { - labelMeta.name -> jsValue - } - /* process tgtVertexId */ - // skip - val tgtVertexIdInnerId = mergedProps.getOrElse(LabelMeta.to, tgtVertexIdRaw.innerId) - - val wal = WalLog( - tsInnerVal.toIdString().toLong, - GraphUtil.fromOp(op), - "edge", - srcVertexId.innerId.toIdString(), - tgtVertexIdInnerId.toIdString(), - labelService(labelWithDir.labelId), - label.label, - Json.toJson(propsJson).toString() - ) - - Option(wal) + val (value, _) = InnerVal.fromBytes(v, 0, v.length, version) + val columnMeta = columnMetas(colId)(propKey) + propsMap += (columnMeta -> value) } } - } catch { - case e: Exception => - println(e.toString) - logger.error(s"$kv", e) - None - } - } - - def snapshotEdgeKeyValueToRow(kv: SKeyValue, - cacheElementOpt: Option[SnapshotEdge], - labelSchema: LabelSchema): Option[WalLog] = { - try { - val LabelSchema(labelService, labels, _, _, labelMetas) = labelSchema - var pos = 0 - val (srcVertexId, srcIdLen) = SourceVertexId.fromBytes(kv.row, pos, kv.row.length, HBaseType.DEFAULT_VERSION) - pos += srcIdLen - - val isTallSchema = pos + 5 != kv.row.length - var tgtVertexId = TargetVertexId(ServiceColumn.Default, srcVertexId.innerId) - - if (isTallSchema) { - val (tgtId, tgtBytesLen) = InnerVal.fromBytes(kv.row, pos, kv.row.length, HBaseType.DEFAULT_VERSION) - tgtVertexId = TargetVertexId(ServiceColumn.Default, tgtId) - pos += tgtBytesLen + val propsJson = for { + (k, v) <- propsMap + jsValue <- JSONParser.innerValToJsValue(v, k.dataType) + } yield { + k.name -> jsValue } - val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4)) - pos += 4 - val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(kv.row, pos) - pos += 1 - - if (!isInverted) None - else { - val label = labels(labelWithDir.labelId) - val schemaVer = label.schemaVersion - - var pos = 0 - val (statusCode, op) = statusCodeWithOp(kv.value(pos)) - pos += 1 - val (props, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer, labelMetas(labelWithDir.labelId)) - val kvsMap = props.toMap - val tsInnerVal = kvsMap(LabelMeta.timestamp).innerVal - - val propsJson = for { - (labelMeta, innerValLikeWithTs) <- props - jsValue <- JSONParser.innerValToJsValue(innerValLikeWithTs.innerVal, labelMeta.dataType) - } yield { - labelMeta.name -> jsValue - } - - pos = endAt - - val wal = WalLog( - tsInnerVal.toIdString().toLong, - GraphUtil.fromOp(op), - "edge", - srcVertexId.innerId.toIdString(), - tgtVertexId.innerId.toIdString(), - labelService(labelWithDir.labelId), - label.label, - Json.toJson(propsJson.toMap).toString() - ) - - Option(wal) - } - } catch { - case e: Exception => - println(e.toString) - logger.error(e) - None + val ts = propsMap + .get(ColumnMeta.timestamp) + .map(_.toIdString().toLong) + .getOrElse(maxTs) + + val walVertex = WalVertex( + ts, + "insert", + "vertex", + vertexInnerId.toIdString(), + columnService(colId).serviceName, + columns(colId).columnName, + Json.toJson(propsJson.toMap).toString() + ) + + Seq(walVertex) } } + } diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala new file mode 100644 index 00000000..8b270634 --- /dev/null +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala @@ -0,0 +1,38 @@ +package org.apache.s2graph.s2jobs.wal.utils + +import org.apache.s2graph.core.{GraphElement, S2Edge, S2Graph, S2Vertex} +import org.apache.s2graph.core.storage.SKeyValue +import org.apache.s2graph.s2jobs.wal.{ColumnSchema, WalLog, WalVertex} + +object SerializeUtil { + private def insertBulkForLoaderAsync(s2: S2Graph, edge: S2Edge, createRelEdges: Boolean = true): Seq[SKeyValue] = { + val relEdges = if (createRelEdges) edge.relatedEdges else List(edge) + + val snapshotEdgeKeyValues = s2.getStorage(edge.toSnapshotEdge.label).serDe.snapshotEdgeSerializer(edge.toSnapshotEdge).toKeyValues + val indexEdgeKeyValues = relEdges.flatMap { edge => + edge.edgesWithIndex.flatMap { indexEdge => + s2.getStorage(indexEdge.label).serDe.indexEdgeSerializer(indexEdge).toKeyValues + } + } + + snapshotEdgeKeyValues ++ indexEdgeKeyValues + } + + def walVertexToKeyValue(walVertex: WalVertex, columnSchema: ColumnSchema): Seq[SKeyValue] = { + + } + def walToKeyValue(walLog: WalLog): Seq[SKeyValue] = { + + } + def toSKeyValues(s2: S2Graph, element: GraphElement, autoEdgeCreate: Boolean = false): Seq[SKeyValue] = { + if (element.isInstanceOf[S2Edge]) { + val edge = element.asInstanceOf[S2Edge] + insertBulkForLoaderAsync(s2, edge, autoEdgeCreate) + } else if (element.isInstanceOf[S2Vertex]) { + val vertex = element.asInstanceOf[S2Vertex] + s2.getStorage(vertex.service).serDe.vertexSerializer(vertex).toKeyValues + } else { + Nil + } + } +} diff --git a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala index ca510003..f53379b8 100644 --- a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala +++ b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala @@ -1,3 +1,4 @@ +/* package org.apache.s2graph.s2jobs.wal.udfs import com.holdenkarau.spark.testing.DataFrameSuiteBase @@ -74,3 +75,4 @@ class HFileParserUDFTest extends FunSuite with Matchers with BeforeAndAfterAll w result.write.mode(SaveMode.Overwrite).json("./parsed.json") } } +*/ diff --git a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala index fa7145f5..43148266 100644 --- a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala +++ b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala @@ -3,7 +3,7 @@ package org.apache.s2graph.s2jobs.wal.utils import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers} class DeserializeUtilTest extends FunSuite with Matchers with BeforeAndAfterAll { - test("test indexEdgeResultToWalsV3.") { + test("test serialize deserialize") { } } From 952cdf68480eb2e1c1a6292102555ea5bdee7d46 Mon Sep 17 00:00:00 2001 From: DO YUNG YOON Date: Sat, 29 Dec 2018 20:31:07 +0900 Subject: [PATCH 07/13] add DeserializeSchema/SerializeSchema. --- .../apache/s2graph/s2jobs/task/Source.scala | 8 +- .../s2graph/s2jobs/wal/ColumnSchema.scala | 26 ++++- .../s2graph/s2jobs/wal/SerializeSchema.scala | 110 ++++++++++++++++++ .../s2jobs/wal/utils/DeserializeUtil.scala | 69 +++++------ .../s2graph/s2jobs/wal/utils/SchemaUtil.scala | 58 ++++++++- .../s2jobs/wal/utils/SerializeUtil.scala | 33 +++++- 6 files changed, 249 insertions(+), 55 deletions(-) create mode 100644 s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SerializeSchema.scala diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala index 55a2a379..8b54ccf8 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala @@ -190,16 +190,16 @@ class S2GraphSource(conf: TaskConf) extends Source(conf) { val cells = HFileGenerator.tableSnapshotDump(ss, config, snapshotPath, restorePath, tableNames, columnFamily, batchSize, labelMapping, buildDegree) - val labelSchema = SchemaUtil.buildLabelSchema(labelNames) - val labelSchemaBCast = sc.broadcast(labelSchema) + val edgeDeserializeSchema = SchemaUtil.buildEdgeDeserializeSchema(labelNames) + val edgeDeserializeSchemaBCast = sc.broadcast(edgeDeserializeSchema) val tallSchemaVersions = Set(HBaseType.VERSION4) val tgtDirection = 0 val results = cells.mapPartitions { iter => - val labelSchema = labelSchemaBCast.value + implicit val edgeDeserializeSchema = edgeDeserializeSchemaBCast.value iter.flatMap { case (_, result) => - DeserializeUtil.indexEdgeResultToWals(result, labelSchema, tallSchemaVersions, tgtDirection) + DeserializeUtil.indexEdgeResultToWals(result, tallSchemaVersions, tgtDirection) } } diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/ColumnSchema.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/ColumnSchema.scala index 9cdc1346..3263e7f7 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/ColumnSchema.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/ColumnSchema.scala @@ -6,8 +6,15 @@ case class ColumnSchema(columnService: Map[Int, Service], columns: Map[Int, ServiceColumn], columnMetas: Map[Int, Map[Int, ColumnMeta]]) { - val services: Set[Service] = columnService.map(_._2).toSet - val serviceColumns: Map[String, Set[ServiceColumn]] = columnService.groupBy(_._2).map { case (service, ls) => + val serviceMap: Map[String, Service] = columnService.map(t => t._2.serviceName -> t._2) + val serviceColumnMap: Map[(String, String), ServiceColumn] = { + columns.map { case (colId, column) => + val key = columnService(colId).serviceName -> column.columnName + + key -> column + } + } + val serviceColumnList: Map[String, Set[ServiceColumn]] = columnService.groupBy(_._2).map { case (service, ls) => val columnIds = ls.map(_._1) service.serviceName -> columnIds.map(columns).toSet @@ -23,4 +30,19 @@ case class ColumnSchema(columnService: Map[Int, Service], (service.serviceName, columnName) -> rev } + def findService(serviceName: String): Service = { + serviceMap(serviceName) + } + + def findServiceColumns(serviceName: String): Set[ServiceColumn] = { + serviceColumnList(serviceName) + } + + def findServiceColumn(serviceName: String, columnName: String): ServiceColumn = { + serviceColumnMap(serviceName -> columnName) + } + + def findServiceColumnMeta(serviceName: String, columnName: String, name: String): ColumnMeta = { + serviceColumnMetas(serviceName -> columnName)(name) + } } diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SerializeSchema.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SerializeSchema.scala new file mode 100644 index 00000000..243fb264 --- /dev/null +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SerializeSchema.scala @@ -0,0 +1,110 @@ +package org.apache.s2graph.s2jobs.wal + +import org.apache.s2graph.core.schema._ + +class DeserializeSchema(columnService: Map[Int, Service], + columns: Map[Int, ServiceColumn], + columnMetas: Map[Int, Map[Byte, ColumnMeta]], + labelService: Map[Int, Service], + labels: Map[Int, Label], + labelIndices: Map[Int, Map[Byte, LabelIndex]], + labelIndexLabelMetas: Map[Int, Map[Byte, Array[LabelMeta]]], + labelMetas: Map[Int, Map[Byte, LabelMeta]]) { + def findColumnService(columnId: Int): Service = { + columnService(columnId) + } + + def findServiceColumn(columnId: Int): ServiceColumn = { + columns(columnId) + } + + def findServiceColumnMeta(columnId: Int, metaSeq: Byte): ColumnMeta = { + columnMetas(columnId)(metaSeq) + } + + def findLabel(labelId: Int): Label = { + labels(labelId) + } + + def findLabelService(labelId: Int): Service = { + labelService(labelId) + } + + def findLabelIndex(labelId: Int, indexSeq: Byte): LabelIndex = { + labelIndices(labelId)(indexSeq) + } + + def findLabelIndexLabelMetas(labelId: Int, indexSeq: Byte): Array[LabelMeta] = { + labelIndexLabelMetas(labelId)(indexSeq) + } + + def findLabelMetas(labelId: Int): Map[Byte, LabelMeta] = { + labelMetas(labelId) + } + + def findLabelMeta(labelId: Int, metaSeq: Byte): LabelMeta = { + labelMetas(labelId)(metaSeq) + } + + def checkLabelExist(labelId: Int): Boolean = { + labels.contains(labelId) + } + + def checkServiceColumnExist(columnId: Int): Boolean = { + columns.contains(columnId) + } +} + +class SerializeSchema(services: Map[String, Service], + serviceColumns: Map[String, Map[String, ServiceColumn]], + serviceColumnSet: Map[String, Set[ServiceColumn]], + serviceColumnMetas: Map[String, Map[String, Map[String, ColumnMeta]]], + labelService: Map[String, Service], + labels: Map[String, Label], + labelIndices: Map[String, Map[String, LabelIndex]], + labelIndexLabelMetas: Map[String, Map[String, Array[LabelMeta]]], + labelMetas: Map[String, Map[String, LabelMeta]]) extends Serializable { + def findService(serviceName: String): Service = { + services(serviceName) + } + + def findServiceColumns(serviceName: String): Set[ServiceColumn] = { + serviceColumnSet(serviceName) + } + + def findServiceColumn(serviceName: String, columnName: String): ServiceColumn = { + serviceColumns(serviceName)(columnName) + } + + def findServiceColumnMeta(serviceName: String, columnName: String, metaName: String): ColumnMeta = { + serviceColumnMetas(serviceName)(columnName)(metaName) + } + + def findLabel(labelName: String): Label = { + labels(labelName) + } + + def findLabelService(labelName: String): Service = { + labelService(labelName) + } + + def findLabelIndex(labelName: String, indexName: String): LabelIndex = { + labelIndices(labelName)(indexName) + } + + def findLabelIndexLabelMetas(labelName: String, indexName: String): Array[LabelMeta] = { + labelIndexLabelMetas(labelName)(indexName) + } + + def findLabelMetas(labelName: String): Map[String, LabelMeta] = { + labelMetas(labelName) + } + + def findLabelMeta(labelName: String, name: String): LabelMeta = { + labelMetas(labelName)(name) + } + + def checkLabelExist(labelName: String) = { + labels.contains(labelName) + } +} diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala index 4154a80d..ff4ad68d 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala @@ -9,7 +9,7 @@ import org.apache.s2graph.core.storage.serde.StorageDeserializable import org.apache.s2graph.core.storage.serde.StorageDeserializable.bytesToInt import org.apache.s2graph.core.types._ import org.apache.s2graph.core.{GraphUtil, JSONParser, S2Vertex} -import org.apache.s2graph.s2jobs.wal.{ColumnSchema, LabelSchema, WalLog, WalVertex} +import org.apache.s2graph.s2jobs.wal._ import org.apache.spark.sql.Row import play.api.libs.json.Json @@ -131,7 +131,7 @@ object DeserializeUtil { schemaVer: String, labelId: Int, labelIdxSeq: Byte, - labelIndexLabelMetas: Map[Int, Map[Byte, Array[LabelMeta]]]): Try[QualifierV3Parsed] = Try { + schema: DeserializeSchema): Try[QualifierV3Parsed] = Try { val (idxPropsRaw, endAt) = StorageDeserializable.bytesToProps(row, offset, schemaVer) val pos = endAt @@ -141,7 +141,7 @@ object DeserializeUtil { val op = row.last - val idxPropsArray = toIndexProps(idxPropsRaw, labelId, labelIdxSeq, labelIndexLabelMetas) + val idxPropsArray = toIndexProps(idxPropsRaw, labelId, labelIdxSeq, schema) QualifierV3Parsed(idxPropsArray, tgtVertexIdRaw, op) } @@ -150,7 +150,7 @@ object DeserializeUtil { schemaVer: String, labelId: Int, labelIdxSeq: Byte, - labelIndexLabelMetas: Map[Int, Map[Byte, Array[LabelMeta]]]): Try[QualifierV3Parsed] = Try { + schema: DeserializeSchema): Try[QualifierV3Parsed] = Try { val (idxPropsRaw, endAt) = StorageDeserializable.bytesToProps(qualifier, 0, schemaVer) @@ -166,7 +166,7 @@ object DeserializeUtil { if (qualifier.length == pos) GraphUtil.defaultOpByte else qualifier.last - val idxPropsArray = toIndexProps(idxPropsRaw, labelId, labelIdxSeq, labelIndexLabelMetas) + val idxPropsArray = toIndexProps(idxPropsRaw, labelId, labelIdxSeq, schema) QualifierV3Parsed(idxPropsArray, tgtVertexIdRaw, op) } @@ -174,9 +174,8 @@ object DeserializeUtil { def toIndexProps(idxPropsRaw: Array[(LabelMeta, InnerValLike)], labelId: Int, labelIdxSeq: Byte, - labelIndexLabelMetas: Map[Int, Map[Byte, Array[LabelMeta]]]): IndexedSeq[(LabelMeta, InnerValLike)] = { - val indexLabelMetas = labelIndexLabelMetas.getOrElse(labelId, throw new IllegalArgumentException(s"${labelId} not found in labelIndexLabelMetas")) - val sortKeyTypesArray = indexLabelMetas.getOrElse(labelIdxSeq, throw new IllegalArgumentException(s"invalid index seq for meta array: ${labelId}, ${labelIdxSeq}")) + schema: DeserializeSchema): IndexedSeq[(LabelMeta, InnerValLike)] = { + val sortKeyTypesArray = schema.findLabelIndexLabelMetas(labelId, labelIdxSeq) val size = idxPropsRaw.length (0 until size).map { ith => @@ -187,10 +186,10 @@ object DeserializeUtil { } def toMetaProps(value: Array[Byte], - labelMetas: Map[Int, Map[Byte, LabelMeta]], labelId: Int, op: Byte, - schemaVer: String) = { + schemaVer: String, + schema: DeserializeSchema) = { /* process props */ if (op == GraphUtil.operations("incrementCount")) { @@ -198,7 +197,7 @@ object DeserializeUtil { val countVal = StorageDeserializable.bytesToLong(value, 0) Array(LabelMeta.count -> InnerVal.withLong(countVal, schemaVer)) } else { - val (props, endAt) = bytesToKeyValues(value, 0, value.length, schemaVer, labelMetas(labelId)) + val (props, endAt) = bytesToKeyValues(value, 0, value.length, schemaVer, schema.findLabelMetas(labelId)) props } } @@ -231,16 +230,14 @@ object DeserializeUtil { def deserializeIndexEdgeCell(cell: Cell, row: Array[Byte], rowV3Parsed: RowV3Parsed, - labelSchema: LabelSchema, - tallSchemaVersions: Set[String]): Option[WalLog] = { - val LabelSchema(labelService, labels, _, labelIndexLabelMetas, labelMetas) = labelSchema - + tallSchemaVersions: Set[String], + schema: DeserializeSchema): Option[WalLog] = { val labelWithDir = rowV3Parsed.labelWithDir val labelId = labelWithDir.labelId val labelIdxSeq = rowV3Parsed.labelIdxSeq val srcVertexId = rowV3Parsed.srcVertexId - val label = labels(labelId) + val label = schema.findLabel(labelId) val schemaVer = label.schemaVersion val isTallSchema = tallSchemaVersions(label.schemaVersion) @@ -253,8 +250,8 @@ object DeserializeUtil { else if (!isTallSchema && !validV3Qualifier) None else { val qualifierParsedTry = - if (isTallSchema) toQualifierV4Parsed(row, rowV3Parsed.pos, schemaVer, labelId, labelIdxSeq, labelIndexLabelMetas) - else toQualifierV3Parsed(qualifier, schemaVer, labelId, labelIdxSeq, labelIndexLabelMetas) + if (isTallSchema) toQualifierV4Parsed(row, rowV3Parsed.pos, schemaVer, labelId, labelIdxSeq, schema) + else toQualifierV3Parsed(qualifier, schemaVer, labelId, labelIdxSeq, schema) qualifierParsedTry.toOption.map { qualifierParsed => val tgtVertexId = qualifierParsed.tgtVertexId @@ -263,7 +260,7 @@ object DeserializeUtil { val value = cell.getValue - val metaPropsArray = toMetaProps(value, labelMetas, labelId, op, schemaVer) + val metaPropsArray = toMetaProps(value, labelId, op, schemaVer, schema) val mergedProps = (idxPropsArray ++ metaPropsArray).toMap val tsInnerVal = mergedProps(LabelMeta.timestamp) @@ -282,7 +279,7 @@ object DeserializeUtil { "edge", srcVertexId.innerId.toIdString(), tgtVertexIdInnerId.toIdString(), - labelService(labelWithDir.labelId), + schema.findLabelService(labelWithDir.labelId).serviceName, label.label, Json.toJson(propsJson).toString() ) @@ -291,15 +288,12 @@ object DeserializeUtil { } def indexEdgeResultToWals(result: Result, - labelSchema: LabelSchema, tallSchemaVersions: Set[String], - tgtDirection: Int = 0): Seq[WalLog] = { + tgtDirection: Int = 0)(implicit schema: DeserializeSchema): Seq[WalLog] = { val rawCells = result.rawCells() if (rawCells.isEmpty) Nil else { - val LabelSchema(_, labels, _, _, _) = labelSchema - val head = rawCells.head val row = head.getRow val rowV3Parsed = toRowV3Parsed(row) @@ -307,12 +301,12 @@ object DeserializeUtil { val labelWithDir = rowV3Parsed.labelWithDir val labelId = labelWithDir.labelId - val inValidRow = rowV3Parsed.isInverted || !labels.contains(labelId) || labelWithDir.dir != tgtDirection + val inValidRow = rowV3Parsed.isInverted || !schema.checkLabelExist(labelId) || labelWithDir.dir != tgtDirection if (inValidRow) Nil else { rawCells.flatMap { cell => - deserializeIndexEdgeCell(cell, row, rowV3Parsed, labelSchema, tallSchemaVersions) + deserializeIndexEdgeCell(cell, row, rowV3Parsed, tallSchemaVersions, schema) } } } @@ -320,13 +314,11 @@ object DeserializeUtil { def snapshotEdgeResultToWals(result: Result, - labelSchema: LabelSchema, - tallSchemaVersions: Set[String]): Seq[WalLog] = { + tallSchemaVersions: Set[String])(implicit schema: DeserializeSchema): Seq[WalLog] = { val rawCells = result.rawCells() if (rawCells.isEmpty) Nil else { - val LabelSchema(labelService, labels, _, _, labelMetas) = labelSchema val head = rawCells.head val row = head.getRow @@ -336,14 +328,14 @@ object DeserializeUtil { if (!isInverted) Nil else { - val label = labels(labelWithDir.labelId) + val label = schema.findLabel(labelWithDir.labelId) val schemaVer = label.schemaVersion rawCells.map { cell => val value = cell.getValue val (_, op) = statusCodeWithOp(value.head) - val (props, _) = bytesToKeyValuesWithTs(value, 1, schemaVer, labelMetas(labelWithDir.labelId)) + val (props, _) = bytesToKeyValuesWithTs(value, 1, schemaVer, schema.findLabelMetas(labelWithDir.labelId)) val kvsMap = props.toMap val tsInnerVal = kvsMap(LabelMeta.timestamp).innerVal @@ -360,7 +352,7 @@ object DeserializeUtil { "edge", srcVertexId.innerId.toIdString(), tgtVertexId.innerId.toIdString(), - labelService(labelWithDir.labelId), + schema.findLabelService(labelWithDir.labelId).serviceName, label.label, Json.toJson(propsJson.toMap).toString() ) @@ -373,12 +365,9 @@ object DeserializeUtil { } def vertexResultToWals(result: Result, - columnSchema: ColumnSchema, - bytesToInt: (Array[Byte], Int) => Int = bytesToInt): Seq[WalVertex] = { + bytesToInt: (Array[Byte], Int) => Int = bytesToInt)(implicit schema: DeserializeSchema): Seq[WalVertex] = { import scala.collection.mutable - val ColumnSchema(columnService, columns, columnMetas) = columnSchema - val rawCells = result.rawCells() if (rawCells.isEmpty) Nil @@ -389,6 +378,8 @@ object DeserializeUtil { val version = HBaseType.DEFAULT_VERSION val (vertexInnerId, colId, _) = VertexId.fromBytesRaw(row, 0, row.length, version) + val column = schema.findServiceColumn(colId) + var maxTs = Long.MinValue val propsMap = mutable.Map.empty[ColumnMeta, InnerValLike] val belongLabelIds = mutable.ListBuffer.empty[Int] @@ -410,7 +401,7 @@ object DeserializeUtil { val v = cell.getValue val (value, _) = InnerVal.fromBytes(v, 0, v.length, version) - val columnMeta = columnMetas(colId)(propKey) + val columnMeta = schema.findServiceColumnMeta(colId, propKey.toByte) propsMap += (columnMeta -> value) } } @@ -432,8 +423,8 @@ object DeserializeUtil { "insert", "vertex", vertexInnerId.toIdString(), - columnService(colId).serviceName, - columns(colId).columnName, + schema.findColumnService(colId).serviceName, + column.columnName, Json.toJson(propsJson.toMap).toString() ) diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala index 0869f8ad..0be6176e 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala @@ -1,9 +1,9 @@ package org.apache.s2graph.s2jobs.wal.utils import com.typesafe.config.Config -import org.apache.s2graph.core.schema.{Label, LabelMeta, Schema} +import org.apache.s2graph.core.schema._ import org.apache.s2graph.core.utils.logger -import org.apache.s2graph.s2jobs.wal.LabelSchema +import org.apache.s2graph.s2jobs.wal.DeserializeSchema object SchemaUtil { @@ -64,14 +64,52 @@ object SchemaUtil { } } - def buildLabelSchema(labelNames: Seq[String]): LabelSchema = { + def buildVertexDeserializeSchema(serviceNameColumnNames: Map[String, Seq[String]]): DeserializeSchema = { + val tree = serviceNameColumnNames.flatMap { case (serviceName, columnNames) => + val service = Service.findByName(serviceName).getOrElse(throw new IllegalArgumentException(s"$serviceName not eixst.")) + + columnNames.map { columnName => + val column = ServiceColumn.find(service.id.get, columnName).getOrElse(throw new IllegalArgumentException(s"$columnName not exist.")) + column -> service + } + } + + val columnService = tree.map { case (column, service) => + column.id.get -> service + } + + val columns = tree.map { case (column, _) => + column.id.get -> column + } + + val columnMetas = columns.map { case (colId, column) => + val innerMap = ColumnMeta.findAllByColumn(colId).map { columnMeta => + columnMeta.seq -> columnMeta + } + + colId -> innerMap.toMap + } + + new DeserializeSchema( + columnService = columnService, + columns = columns, + columnMetas = columnMetas, + labelService = Map.empty, + labels = Map.empty, + labelIndices = Map.empty, + labelIndexLabelMetas = Map.empty, + labelMetas = Map.empty + ) + } + + def buildEdgeDeserializeSchema(labelNames: Seq[String]): DeserializeSchema = { val labels = labelNames.map { labelName => val label = Label.findByName(labelName).getOrElse(throw new IllegalArgumentException(s"$labelName not exist.")) label.id.get -> label }.toMap - val labelServices = labels.map { case (id, label) => - id -> label.serviceName + val labelService = labels.map { case (id, label) => + id -> label.service } val labelIndices = labels.map { case (id, label) => @@ -91,6 +129,14 @@ object SchemaUtil { id -> m } - LabelSchema(labelServices, labels, labelIndices, labelIndexLabelMetas, labelMetas) + new DeserializeSchema( + columnService = Map.empty, + columns = Map.empty, + columnMetas = Map.empty, + labelService = labelService, + labels = labels, + labelIndices = labelIndices, + labelIndexLabelMetas = labelIndexLabelMetas, + labelMetas = labelMetas) } } diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala index 8b270634..fbb5d024 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala @@ -1,8 +1,13 @@ package org.apache.s2graph.s2jobs.wal.utils -import org.apache.s2graph.core.{GraphElement, S2Edge, S2Graph, S2Vertex} +import org.apache.s2graph.core._ +import org.apache.s2graph.core.schema.ColumnMeta import org.apache.s2graph.core.storage.SKeyValue +import org.apache.s2graph.core.storage.serde.Serializable +import org.apache.s2graph.core.storage.serde.StorageSerializable.intToBytes +import org.apache.s2graph.core.types.VertexId import org.apache.s2graph.s2jobs.wal.{ColumnSchema, WalLog, WalVertex} +import play.api.libs.json.{JsObject, Json} object SerializeUtil { private def insertBulkForLoaderAsync(s2: S2Graph, edge: S2Edge, createRelEdges: Boolean = true): Seq[SKeyValue] = { @@ -18,12 +23,32 @@ object SerializeUtil { snapshotEdgeKeyValues ++ indexEdgeKeyValues } - def walVertexToKeyValue(walVertex: WalVertex, columnSchema: ColumnSchema): Seq[SKeyValue] = { + def walVertexToKeyValue(v: WalVertex)(implicit schema: ColumnSchema): Seq[SKeyValue] = { + val service = schema.findService(v.service) + val column = schema.findServiceColumn(v.service, v.column) - } - def walToKeyValue(walLog: WalLog): Seq[SKeyValue] = { + val table = service.hTableName.getBytes("UTF-8") + val ts = v.timestamp + val cf = Serializable.vertexCf + + val innerId = JSONParser.toInnerVal(v.id, column.columnType, column.schemaVersion) + val vertexId = new VertexId(column, innerId) + val row = vertexId.bytes + + val propsJson = Json.parse(v.props).as[JsObject] +// ++ Json.obj(ColumnMeta.lastModifiedAtColumn.name -> ts) + propsJson.fields.flatMap { case (key, jsValue) => + val columnMeta = schema.findServiceColumnMeta(v.service, v.column, key) + + JSONParser.jsValueToInnerVal(jsValue, columnMeta.dataType, column.schemaVersion).map { innerVal => + val qualifier = intToBytes(columnMeta.seq) + val value = innerVal.bytes + SKeyValue(table, row, cf, qualifier, value, ts) + } + } } + def toSKeyValues(s2: S2Graph, element: GraphElement, autoEdgeCreate: Boolean = false): Seq[SKeyValue] = { if (element.isInstanceOf[S2Edge]) { val edge = element.asInstanceOf[S2Edge] From c750774202390f00a58aa279b7cfe2245248699f Mon Sep 17 00:00:00 2001 From: DO YUNG YOON Date: Sun, 30 Dec 2018 14:59:47 +0900 Subject: [PATCH 08/13] merge DeserializeSchema and SerializeSchema to SchemaManager. --- .../core/storage/serde/Serializable.scala | 1 + .../apache/s2graph/s2jobs/task/Source.scala | 8 +- .../s2graph/s2jobs/wal/SchemaManager.scala | 247 ++++++++++++++++++ .../s2graph/s2jobs/wal/SerializeSchema.scala | 110 -------- .../s2jobs/wal/utils/DeserializeUtil.scala | 22 +- .../s2graph/s2jobs/wal/utils/SchemaUtil.scala | 83 ++---- .../s2jobs/wal/utils/SerializeUtil.scala | 225 ++++++++++++++-- .../wal/utils/DeserializeUtilTest.scala | 47 ++++ 8 files changed, 539 insertions(+), 204 deletions(-) create mode 100644 s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SchemaManager.scala delete mode 100644 s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SerializeSchema.scala diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/Serializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/Serializable.scala index 46b4860e..d51ee77c 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/Serializable.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/Serializable.scala @@ -22,6 +22,7 @@ package org.apache.s2graph.core.storage.serde object Serializable { val vertexCf = "v".getBytes("UTF-8") val edgeCf = "e".getBytes("UTF-8") + val snapshotEdgeCf = "s".getBytes("UTF-8") } trait Serializable[E] extends StorageSerializable[E] diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala index 8b54ccf8..381c94df 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala @@ -190,16 +190,16 @@ class S2GraphSource(conf: TaskConf) extends Source(conf) { val cells = HFileGenerator.tableSnapshotDump(ss, config, snapshotPath, restorePath, tableNames, columnFamily, batchSize, labelMapping, buildDegree) - val edgeDeserializeSchema = SchemaUtil.buildEdgeDeserializeSchema(labelNames) - val edgeDeserializeSchemaBCast = sc.broadcast(edgeDeserializeSchema) + val schema = SchemaUtil.buildEdgeDeserializeSchema(labelNames) + val schemaBCast = sc.broadcast(schema) val tallSchemaVersions = Set(HBaseType.VERSION4) val tgtDirection = 0 val results = cells.mapPartitions { iter => - implicit val edgeDeserializeSchema = edgeDeserializeSchemaBCast.value + val schema = schemaBCast.value iter.flatMap { case (_, result) => - DeserializeUtil.indexEdgeResultToWals(result, tallSchemaVersions, tgtDirection) + DeserializeUtil.indexEdgeResultToWals(result, schema, tallSchemaVersions, tgtDirection) } } diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SchemaManager.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SchemaManager.scala new file mode 100644 index 00000000..70508daf --- /dev/null +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SchemaManager.scala @@ -0,0 +1,247 @@ +package org.apache.s2graph.s2jobs.wal + +import org.apache.s2graph.core.schema._ + +class SchemaManager(serviceLs: Seq[Service], + serviceColumnLs: Seq[ServiceColumn], + columnMetaLs: Seq[ColumnMeta], + labelLs: Seq[Label], + labelIndexLs: Seq[LabelIndex], + labelMetaLs: Seq[LabelMeta]) { + private val services = serviceLs.map { service => service.serviceName -> service }.toMap + private val servicesInverted = serviceLs.map { service => service.id.get -> service }.toMap + + private val columnServiceMap: Map[ServiceColumn, Service] = serviceColumnLs.flatMap { column => + servicesInverted.get(column.serviceId).map(column -> _) + }.toMap + +// private val columnService = columnServiceMap.map(t => t._1.columnName -> t._2) + private val columnServiceInverted = columnServiceMap.map(t => t._1.id.get -> t._2) + +// private val columns = columnServiceMap.map(t => t._1.columnName -> t._1) + private val columnsInverted = columnServiceMap.map(t => t._1.id.get -> t._1) + + private val columnMetasMap: Map[ServiceColumn, Seq[ColumnMeta]] = + columnMetaLs + .groupBy(_.columnId) + .flatMap { case (columnId, metas) => + columnsInverted.get(columnId).map(_ -> (metas ++ ColumnMeta.reservedMetas)) + } + +// private val columnMetas = columnMetasMap.map { case (column, metas) => +// val innerMap = metas.map { meta => +// meta.name -> meta +// }.toMap +// +// column.columnName -> innerMap +// } + + private val columnMetasInverted = columnMetasMap.map { case (column, metas) => + val innerMap = metas.map { meta => + meta.seq -> meta + }.toMap + + column.id.get -> innerMap + } + + private val labelsMap = labelLs.flatMap { label => + services.get(label.serviceName).map(label -> _) + }.toMap + + private val labelService = labelsMap.map(t => t._1.label -> t._2) + private val labelServiceInverted = labelsMap.map(t => t._1.id.get -> t._2) + + private val labels = labelsMap.map(t => t._1.label -> t._1) + private val labelsInverted = labelsMap.map(t => t._1.id.get -> t._1) + + private val labelMetasMap: Map[Label, Seq[LabelMeta]] = + labelMetaLs + .groupBy(_.labelId) + .flatMap { case (labelId, metas) => + labelsInverted.get(labelId).map(_ -> (metas ++ LabelMeta.reservedMetas)) + } + + private val labelMetas = labelMetasMap.map { case (label, metas) => + val innerMap = metas.map { meta => + meta.name -> meta + }.toMap + + label.label -> innerMap + } + private val labelMetasInverted = labelMetasMap.map { case (label, metas) => + val innerMap = metas.map { meta => + meta.seq -> meta + }.toMap + + label.id.get -> innerMap + } + + private val labelIndicesMap = labelIndexLs + .groupBy(_.labelId) + .flatMap { case (labelId, indices) => + labelsInverted.get(labelId).map(_ -> indices) + } + + private val labelIndices = labelIndicesMap.map { case (label, indices) => + val innerMap = indices.map { index => + index.name -> index + }.toMap + + label.label -> innerMap + } + + private val labelIndicesInverted = labelIndicesMap.map { case (label, indices) => + val innerMap = indices.map { index => + index.seq -> index + }.toMap + + label.id.get -> innerMap + } + + private val labelIndexLabelMetas = labelIndicesMap.map { case (label, indices) => + val innerMap = indices.map { index => + val metas = index.metaSeqs.map { seq => + labelMetasInverted(label.id.get)(seq) + }.toArray + + index.name -> metas + }.toMap + + label.label -> innerMap + } + + private val labelIndexLabelMetasInverted = labelIndicesMap.map { case (label, indices) => + val innerMap = indices.map { index => + val metas = index.metaSeqs.map { seq => + labelMetasInverted(label.id.get)(seq) + }.toArray + + index.seq -> metas + }.toMap + + label.id.get -> innerMap + } + + private val serviceIdColumnNameColumn = columnServiceMap.map { case (column, service) => + val key = service.id.get -> column.columnName + key -> column + } + + private val serviceNameColumnNameColumn = columnServiceMap.map { case (column, service) => + val key = service.serviceName -> column.columnName + key -> column + } + + def findColumnService(columnId: Int): Service = { + columnServiceInverted(columnId) + } + + def findServiceColumn(columnId: Int): ServiceColumn = { + columnsInverted(columnId) + } + + def findServiceColumnMeta(columnId: Int, metaSeq: Byte): ColumnMeta = { + columnMetasInverted(columnId)(metaSeq) + } + + def findLabel(labelId: Int): Label = { + labelsInverted(labelId) + } + + def findLabelService(labelId: Int): Service = { + labelServiceInverted(labelId) + } + + def findLabelIndex(labelId: Int, indexSeq: Byte): LabelIndex = { + labelIndicesInverted(labelId)(indexSeq) + } + + def findLabelIndexLabelMetas(labelId: Int, indexSeq: Byte): Array[LabelMeta] = { + labelIndexLabelMetasInverted(labelId)(indexSeq) + } + + def findLabelMetas(labelId: Int): Map[Byte, LabelMeta] = { + labelMetasInverted(labelId) + } + + def findLabelMeta(labelId: Int, metaSeq: Byte): LabelMeta = { + labelMetasInverted(labelId)(metaSeq) + } + + def checkLabelExist(labelId: Int): Boolean = { + labelsInverted.contains(labelId) + } + + def checkServiceColumnExist(columnId: Int): Boolean = { + columnsInverted.contains(columnId) + } + + def findServiceColumn(serviceName: String, columnName: String): ServiceColumn = { + serviceNameColumnNameColumn(serviceName -> columnName) + } + + def findLabel(labelName: String): Label = { + labels(labelName) + } + + def findLabelService(labelName: String): Service = { + labelService(labelName) + } + + def findLabelIndices(label: Label): Map[String, LabelIndex] = { + findLabelIndices(label.label) + } + + def findLabelIndices(labelName: String): Map[String, LabelIndex] = { + labelIndices(labelName) + } + + def findLabelIndex(labelName: String, indexName: String): LabelIndex = { + labelIndices(labelName)(indexName) + } + + def findLabelIndexLabelMetas(labelName: String, indexName: String): Array[LabelMeta] = { + labelIndexLabelMetas(labelName)(indexName) + } + + def findLabelMetas(labelName: String): Map[String, LabelMeta] = { + labelMetas(labelName) + } + + def findLabelMeta(labelName: String, name: String): LabelMeta = { + labelMetas(labelName)(name) + } + + def checkLabelExist(labelName: String): Boolean = { + labels.contains(labelName) + } + + def findSrcServiceColumn(label: Label): ServiceColumn = { + serviceIdColumnNameColumn(label.srcServiceId -> label.srcColumnName) + } + + def findTgtServiceColumn(label: Label): ServiceColumn = { + serviceIdColumnNameColumn(label.tgtServiceId -> label.tgtColumnName) + } + + def findSrcServiceColumn(labelName: String): ServiceColumn = { + val label = findLabel(labelName) + serviceIdColumnNameColumn(label.srcServiceId -> label.srcColumnName) + } + + def findTgtServiceColumn(labelName: String): ServiceColumn = { + val label = findLabel(labelName) + serviceIdColumnNameColumn(label.tgtServiceId -> label.tgtColumnName) + } +} + +object SchemaManager { + def apply(serviceLs: Seq[Service], + serviceColumnLs: Seq[ServiceColumn], + columnMetaLs: Seq[ColumnMeta], + labelLs: Seq[Label], + labelIndexLs: Seq[LabelIndex], + labelMetaLs: Seq[LabelMeta]): SchemaManager = { + new SchemaManager(serviceLs, serviceColumnLs, columnMetaLs, labelLs, labelIndexLs, labelMetaLs) + } +} diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SerializeSchema.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SerializeSchema.scala deleted file mode 100644 index 243fb264..00000000 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SerializeSchema.scala +++ /dev/null @@ -1,110 +0,0 @@ -package org.apache.s2graph.s2jobs.wal - -import org.apache.s2graph.core.schema._ - -class DeserializeSchema(columnService: Map[Int, Service], - columns: Map[Int, ServiceColumn], - columnMetas: Map[Int, Map[Byte, ColumnMeta]], - labelService: Map[Int, Service], - labels: Map[Int, Label], - labelIndices: Map[Int, Map[Byte, LabelIndex]], - labelIndexLabelMetas: Map[Int, Map[Byte, Array[LabelMeta]]], - labelMetas: Map[Int, Map[Byte, LabelMeta]]) { - def findColumnService(columnId: Int): Service = { - columnService(columnId) - } - - def findServiceColumn(columnId: Int): ServiceColumn = { - columns(columnId) - } - - def findServiceColumnMeta(columnId: Int, metaSeq: Byte): ColumnMeta = { - columnMetas(columnId)(metaSeq) - } - - def findLabel(labelId: Int): Label = { - labels(labelId) - } - - def findLabelService(labelId: Int): Service = { - labelService(labelId) - } - - def findLabelIndex(labelId: Int, indexSeq: Byte): LabelIndex = { - labelIndices(labelId)(indexSeq) - } - - def findLabelIndexLabelMetas(labelId: Int, indexSeq: Byte): Array[LabelMeta] = { - labelIndexLabelMetas(labelId)(indexSeq) - } - - def findLabelMetas(labelId: Int): Map[Byte, LabelMeta] = { - labelMetas(labelId) - } - - def findLabelMeta(labelId: Int, metaSeq: Byte): LabelMeta = { - labelMetas(labelId)(metaSeq) - } - - def checkLabelExist(labelId: Int): Boolean = { - labels.contains(labelId) - } - - def checkServiceColumnExist(columnId: Int): Boolean = { - columns.contains(columnId) - } -} - -class SerializeSchema(services: Map[String, Service], - serviceColumns: Map[String, Map[String, ServiceColumn]], - serviceColumnSet: Map[String, Set[ServiceColumn]], - serviceColumnMetas: Map[String, Map[String, Map[String, ColumnMeta]]], - labelService: Map[String, Service], - labels: Map[String, Label], - labelIndices: Map[String, Map[String, LabelIndex]], - labelIndexLabelMetas: Map[String, Map[String, Array[LabelMeta]]], - labelMetas: Map[String, Map[String, LabelMeta]]) extends Serializable { - def findService(serviceName: String): Service = { - services(serviceName) - } - - def findServiceColumns(serviceName: String): Set[ServiceColumn] = { - serviceColumnSet(serviceName) - } - - def findServiceColumn(serviceName: String, columnName: String): ServiceColumn = { - serviceColumns(serviceName)(columnName) - } - - def findServiceColumnMeta(serviceName: String, columnName: String, metaName: String): ColumnMeta = { - serviceColumnMetas(serviceName)(columnName)(metaName) - } - - def findLabel(labelName: String): Label = { - labels(labelName) - } - - def findLabelService(labelName: String): Service = { - labelService(labelName) - } - - def findLabelIndex(labelName: String, indexName: String): LabelIndex = { - labelIndices(labelName)(indexName) - } - - def findLabelIndexLabelMetas(labelName: String, indexName: String): Array[LabelMeta] = { - labelIndexLabelMetas(labelName)(indexName) - } - - def findLabelMetas(labelName: String): Map[String, LabelMeta] = { - labelMetas(labelName) - } - - def findLabelMeta(labelName: String, name: String): LabelMeta = { - labelMetas(labelName)(name) - } - - def checkLabelExist(labelName: String) = { - labels.contains(labelName) - } -} diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala index ff4ad68d..074923a1 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala @@ -4,7 +4,7 @@ import org.apache.hadoop.hbase.client.Result import org.apache.hadoop.hbase.util.Bytes import org.apache.hadoop.hbase.{Cell, CellUtil} import org.apache.s2graph.core.schema.{ColumnMeta, LabelMeta, ServiceColumn} -import org.apache.s2graph.core.storage.{CanSKeyValue, SKeyValue} +import org.apache.s2graph.core.storage.SKeyValue import org.apache.s2graph.core.storage.serde.StorageDeserializable import org.apache.s2graph.core.storage.serde.StorageDeserializable.bytesToInt import org.apache.s2graph.core.types._ @@ -13,7 +13,6 @@ import org.apache.s2graph.s2jobs.wal._ import org.apache.spark.sql.Row import play.api.libs.json.Json -import scala.collection.mutable.ListBuffer import scala.util.{Failure, Success, Try} object DeserializeUtil { @@ -131,7 +130,7 @@ object DeserializeUtil { schemaVer: String, labelId: Int, labelIdxSeq: Byte, - schema: DeserializeSchema): Try[QualifierV3Parsed] = Try { + schema: SchemaManager): Try[QualifierV3Parsed] = Try { val (idxPropsRaw, endAt) = StorageDeserializable.bytesToProps(row, offset, schemaVer) val pos = endAt @@ -150,7 +149,7 @@ object DeserializeUtil { schemaVer: String, labelId: Int, labelIdxSeq: Byte, - schema: DeserializeSchema): Try[QualifierV3Parsed] = Try { + schema: SchemaManager): Try[QualifierV3Parsed] = Try { val (idxPropsRaw, endAt) = StorageDeserializable.bytesToProps(qualifier, 0, schemaVer) @@ -174,7 +173,7 @@ object DeserializeUtil { def toIndexProps(idxPropsRaw: Array[(LabelMeta, InnerValLike)], labelId: Int, labelIdxSeq: Byte, - schema: DeserializeSchema): IndexedSeq[(LabelMeta, InnerValLike)] = { + schema: SchemaManager): IndexedSeq[(LabelMeta, InnerValLike)] = { val sortKeyTypesArray = schema.findLabelIndexLabelMetas(labelId, labelIdxSeq) val size = idxPropsRaw.length @@ -189,7 +188,7 @@ object DeserializeUtil { labelId: Int, op: Byte, schemaVer: String, - schema: DeserializeSchema) = { + schema: SchemaManager) = { /* process props */ if (op == GraphUtil.operations("incrementCount")) { @@ -231,7 +230,7 @@ object DeserializeUtil { row: Array[Byte], rowV3Parsed: RowV3Parsed, tallSchemaVersions: Set[String], - schema: DeserializeSchema): Option[WalLog] = { + schema: SchemaManager): Option[WalLog] = { val labelWithDir = rowV3Parsed.labelWithDir val labelId = labelWithDir.labelId val labelIdxSeq = rowV3Parsed.labelIdxSeq @@ -288,8 +287,9 @@ object DeserializeUtil { } def indexEdgeResultToWals(result: Result, + schema: SchemaManager, tallSchemaVersions: Set[String], - tgtDirection: Int = 0)(implicit schema: DeserializeSchema): Seq[WalLog] = { + tgtDirection: Int = 0): Seq[WalLog] = { val rawCells = result.rawCells() if (rawCells.isEmpty) Nil @@ -314,7 +314,8 @@ object DeserializeUtil { def snapshotEdgeResultToWals(result: Result, - tallSchemaVersions: Set[String])(implicit schema: DeserializeSchema): Seq[WalLog] = { + schema: SchemaManager, + tallSchemaVersions: Set[String]): Seq[WalLog] = { val rawCells = result.rawCells() if (rawCells.isEmpty) Nil @@ -365,7 +366,8 @@ object DeserializeUtil { } def vertexResultToWals(result: Result, - bytesToInt: (Array[Byte], Int) => Int = bytesToInt)(implicit schema: DeserializeSchema): Seq[WalVertex] = { + schema: SchemaManager, + bytesToInt: (Array[Byte], Int) => Int = bytesToInt): Seq[WalVertex] = { import scala.collection.mutable val rawCells = result.rawCells() diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala index 0be6176e..c65e6e6c 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala @@ -3,7 +3,7 @@ package org.apache.s2graph.s2jobs.wal.utils import com.typesafe.config.Config import org.apache.s2graph.core.schema._ import org.apache.s2graph.core.utils.logger -import org.apache.s2graph.s2jobs.wal.DeserializeSchema +import org.apache.s2graph.s2jobs.wal.SchemaManager object SchemaUtil { @@ -64,79 +64,46 @@ object SchemaUtil { } } - def buildVertexDeserializeSchema(serviceNameColumnNames: Map[String, Seq[String]]): DeserializeSchema = { - val tree = serviceNameColumnNames.flatMap { case (serviceName, columnNames) => - val service = Service.findByName(serviceName).getOrElse(throw new IllegalArgumentException(s"$serviceName not eixst.")) - - columnNames.map { columnName => - val column = ServiceColumn.find(service.id.get, columnName).getOrElse(throw new IllegalArgumentException(s"$columnName not exist.")) - column -> service - } + def buildVertexDeserializeSchema(serviceNameColumnNames: Map[String, Seq[String]]): SchemaManager = { + val serviceLs = serviceNameColumnNames.keys.toSeq.map { serviceName => + Service.findByName(serviceName).getOrElse(throw new IllegalArgumentException(s"$serviceName not found.")) } - val columnService = tree.map { case (column, service) => - column.id.get -> service + val serviceColumnLs = serviceLs.flatMap { service => + ServiceColumn.findByServiceId(service.id.get) } - val columns = tree.map { case (column, _) => - column.id.get -> column + val columnMetaLs = serviceColumnLs.flatMap { column => + column.metas } - val columnMetas = columns.map { case (colId, column) => - val innerMap = ColumnMeta.findAllByColumn(colId).map { columnMeta => - columnMeta.seq -> columnMeta - } + SchemaManager(serviceLs, serviceColumnLs, columnMetaLs, Nil, Nil, Nil) + } - colId -> innerMap.toMap + def buildEdgeDeserializeSchema(labelNames: Seq[String]): SchemaManager = { + val labelLs = labelNames.map { labelName => + Label.findByName(labelName).getOrElse(throw new IllegalArgumentException(s"$labelName not exist.")) } - new DeserializeSchema( - columnService = columnService, - columns = columns, - columnMetas = columnMetas, - labelService = Map.empty, - labels = Map.empty, - labelIndices = Map.empty, - labelIndexLabelMetas = Map.empty, - labelMetas = Map.empty - ) - } - - def buildEdgeDeserializeSchema(labelNames: Seq[String]): DeserializeSchema = { - val labels = labelNames.map { labelName => - val label = Label.findByName(labelName).getOrElse(throw new IllegalArgumentException(s"$labelName not exist.")) - label.id.get -> label - }.toMap + val serviceLs = labelLs.flatMap { label => + Seq(label.srcService, label.tgtService) + }.distinct - val labelService = labels.map { case (id, label) => - id -> label.service + val serviceColumnLs = serviceLs.flatMap { service => + ServiceColumn.findByServiceId(service.id.get) } - val labelIndices = labels.map { case (id, label) => - id -> label.indicesMap + val columnMetaLs = serviceColumnLs.flatMap { column => + column.metas } - val labelMetas = labels.map { case (id, label) => - id -> labelMetaMap(labels(id)) + val labelIndexLs = labelLs.flatMap { label => + label.indices } - - val labelIndexLabelMetas = labelIndices.map { case (id, indicesMap) => - val metaPropsMap = labelMetas(id) - val m = indicesMap.map { case (idxSeq, labelIndex) => - idxSeq -> labelIndex.metaSeqs.flatMap(seq => metaPropsMap.get(seq)).toArray - } - - id -> m + val labelMetaLs = labelLs.flatMap { label => + label.metaProps } - new DeserializeSchema( - columnService = Map.empty, - columns = Map.empty, - columnMetas = Map.empty, - labelService = labelService, - labels = labels, - labelIndices = labelIndices, - labelIndexLabelMetas = labelIndexLabelMetas, - labelMetas = labelMetas) + SchemaManager(serviceLs, serviceColumnLs, columnMetaLs, labelLs, labelIndexLs, labelMetaLs) } } diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala index fbb5d024..974847bc 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala @@ -1,15 +1,17 @@ package org.apache.s2graph.s2jobs.wal.utils +import org.apache.hadoop.hbase.util.Bytes import org.apache.s2graph.core._ -import org.apache.s2graph.core.schema.ColumnMeta +import org.apache.s2graph.core.schema.{Label, LabelIndex, LabelMeta, ServiceColumn} import org.apache.s2graph.core.storage.SKeyValue import org.apache.s2graph.core.storage.serde.Serializable -import org.apache.s2graph.core.storage.serde.StorageSerializable.intToBytes -import org.apache.s2graph.core.types.VertexId -import org.apache.s2graph.s2jobs.wal.{ColumnSchema, WalLog, WalVertex} -import play.api.libs.json.{JsObject, Json} +import org.apache.s2graph.core.types._ +import org.apache.s2graph.s2jobs.wal.{SchemaManager, WalLog} object SerializeUtil { + + import org.apache.s2graph.core.storage.serde.StorageSerializable._ + private def insertBulkForLoaderAsync(s2: S2Graph, edge: S2Edge, createRelEdges: Boolean = true): Seq[SKeyValue] = { val relEdges = if (createRelEdges) edge.relatedEdges else List(edge) @@ -23,30 +25,209 @@ object SerializeUtil { snapshotEdgeKeyValues ++ indexEdgeKeyValues } - def walVertexToKeyValue(v: WalVertex)(implicit schema: ColumnSchema): Seq[SKeyValue] = { - val service = schema.findService(v.service) - val column = schema.findServiceColumn(v.service, v.column) + def buildIndexInnerProps(walLog: WalLog, + innerProps: Map[LabelMeta, InnerValLike], + defaultProps: Map[LabelMeta, InnerValLike], + label: Label, + labelMetasInOrder: Array[LabelMeta]): Array[(LabelMeta, InnerValLike)] = { + labelMetasInOrder.map { labelMeta => + innerProps.get(labelMeta) match { + case None => + val defaultVal = JSONParser.toInnerVal(labelMeta.defaultValue, labelMeta.dataType, label.schemaVersion) + val innerVal = defaultProps.getOrElse(labelMeta, defaultVal) + + labelMeta -> innerVal + case Some(innerVal) => labelMeta -> innerVal + } + } + } + + def buildInnerDefaultProps(walLog: WalLog, + label: Label, + srcColumn: ServiceColumn, + tgtColumn: ServiceColumn): Map[LabelMeta, InnerValLike] = { + Map( + LabelMeta.timestamp -> InnerVal.withLong(walLog.timestamp, label.schemaVersion), + LabelMeta.from -> JSONParser.toInnerVal(walLog.from, srcColumn.columnType, srcColumn.schemaVersion), + LabelMeta.to -> JSONParser.toInnerVal(walLog.to, tgtColumn.columnType, tgtColumn.schemaVersion) + ) + } + + def buildInnerProps(walLog: WalLog, + label: Label, + schema: SchemaManager): Map[LabelMeta, InnerValLike] = { + val labelMetas = schema.findLabelMetas(walLog.label) + val props = walLog.propsJson.fields.flatMap { case (key, jsValue) => + labelMetas.get(key).flatMap { labelMeta => + JSONParser.jsValueToInnerVal(jsValue, labelMeta.dataType, label.schemaVersion).map { innerVal => + labelMeta -> innerVal + } + } + }.toMap + + props ++ Map(LabelMeta.timestamp -> InnerVal.withLong(walLog.timestamp, label.schemaVersion)) + } + + case class PartialResult(label: Label, + srcColumn: ServiceColumn, + srcVertexId: VertexId, + tgtColumn: ServiceColumn, + tgtVertexId: VertexId, + op: Byte, + dir: Int, + props: Map[LabelMeta, InnerValLike], + defaultProps: Map[LabelMeta, InnerValLike]) { + val schemaVer = label.schemaVersion + + // need to build Array[Byte] for SKeyValue + lazy val table = label.hTableName.getBytes("UTF-8") + lazy val cf = Serializable.edgeCf + lazy val labelWithDir = LabelWithDirection(label.id.get, dir) + lazy val labelWithDirBytes = labelWithDir.bytes + lazy val srcIdBytes = VertexId.toSourceVertexId(srcVertexId).bytes + lazy val tgtIdBytes = VertexId.toTargetVertexId(tgtVertexId).bytes + } + + def buildPartialResult(walLog: WalLog, + schema: SchemaManager): PartialResult = { + val label = schema.findLabel(walLog.label) + val dir = 0 + + val srcColumn = schema.findSrcServiceColumn(label) + val srcInnerVal = JSONParser.toInnerVal(walLog.from, srcColumn.columnType, srcColumn.schemaVersion) + val srcVertexId = VertexId(srcColumn, srcInnerVal) + + val tgtColumn = schema.findTgtServiceColumn(label) + val tgtInnerVal = JSONParser.toInnerVal(walLog.to, tgtColumn.columnType, tgtColumn.schemaVersion) + val tgtVertexId = VertexId(tgtColumn, tgtInnerVal) + + val allProps = buildInnerProps(walLog, label, schema) + val defaultProps = buildInnerDefaultProps(walLog, label, srcColumn, tgtColumn) + + val op = GraphUtil.toOp(walLog.operation).getOrElse(throw new IllegalArgumentException(s"${walLog.operation} operation is not supported.")) + + PartialResult(label, srcColumn, srcVertexId, tgtColumn, tgtVertexId, op, dir, allProps, defaultProps) + } + + def walToIndexEdgeKeyValue(walLog: WalLog, + schema: SchemaManager, + tallSchemaVersions: Set[String]): Iterable[SKeyValue] = { + //TODO: Degree Edge is not considered here. + val pr = buildPartialResult(walLog, schema) + val label = pr.label + val srcIdBytes = pr.srcIdBytes + val tgtIdBytes = pr.tgtIdBytes + val labelWithDirBytes = pr.labelWithDirBytes + val allProps = pr.props + val defaultProps = pr.defaultProps + val op = pr.op + val table = pr.table + val cf = pr.cf + val isTallSchema = tallSchemaVersions(label.schemaVersion) + + schema.findLabelIndices(label).map { case (indexName, index) => + val labelMetasInOrder = schema.findLabelIndexLabelMetas(label.label, indexName) + + val orderedProps = buildIndexInnerProps(walLog, allProps, defaultProps, label, labelMetasInOrder) + val idxPropsMap = orderedProps.toMap + val idxPropsBytes = propsToBytes(orderedProps) + val metaProps = allProps.filterKeys(m => !idxPropsMap.contains(m)) - val table = service.hTableName.getBytes("UTF-8") - val ts = v.timestamp - val cf = Serializable.vertexCf + val row = + if (isTallSchema) { + val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(index.seq, isInverted = false) - val innerId = JSONParser.toInnerVal(v.id, column.columnType, column.schemaVersion) - val vertexId = new VertexId(column, innerId) - val row = vertexId.bytes + val row = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes) - val propsJson = Json.parse(v.props).as[JsObject] -// ++ Json.obj(ColumnMeta.lastModifiedAtColumn.name -> ts) - propsJson.fields.flatMap { case (key, jsValue) => - val columnMeta = schema.findServiceColumnMeta(v.service, v.column, key) + val qualifier = idxPropsMap.get(LabelMeta.to) match { + case None => Bytes.add(idxPropsBytes, tgtIdBytes) + case Some(_) => idxPropsBytes + } - JSONParser.jsValueToInnerVal(jsValue, columnMeta.dataType, column.schemaVersion).map { innerVal => - val qualifier = intToBytes(columnMeta.seq) - val value = innerVal.bytes + val opByte = + if (op == GraphUtil.operations("incrementCount")) op + else GraphUtil.defaultOpByte - SKeyValue(table, row, cf, qualifier, value, ts) + Bytes.add(row, qualifier, Array.fill(1)(opByte)) + } else { + val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(index.seq, isInverted = false) + Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes) + } + + val qualifier = + if (isTallSchema) Array.empty[Byte] + else { + if (op == GraphUtil.operations("incrementCount")) { + Bytes.add(idxPropsBytes, tgtIdBytes, Array.fill(1)(op)) + } else { + idxPropsMap.get(LabelMeta.to) match { + case None => Bytes.add(idxPropsBytes, tgtIdBytes) + case Some(_) => idxPropsBytes + } + } + } + + val value = { + if (op == GraphUtil.operations("incrementCount")) { + longToBytes(allProps(LabelMeta.count).toString().toLong) + } else { + propsToKeyValues(metaProps.toSeq) + } } + + SKeyValue(table, row, cf, qualifier, value, walLog.timestamp, op, durability = true) + } + } + + def walToSnapshotEdgeKeyValue(walLog: WalLog, + schema: SchemaManager, + tallSchemaVersions: Set[String]): Iterable[SKeyValue] = { + val ts = walLog.timestamp + val pr = buildPartialResult(walLog, schema) + val label = pr.label + + val labelWithDirBytes = pr.labelWithDirBytes + val allProps = pr.props.map { case (labelMeta, innerVal) => + labelMeta.seq -> InnerValLikeWithTs(innerVal, ts) + } + + val op = pr.op + val table = pr.table + val cf = pr.cf + val statusCode = 0.toByte + + val isTallSchema = tallSchemaVersions(label.schemaVersion) + + val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(LabelIndex.DefaultSeq, isInverted = true) + val rowFirstBytes = + if (isTallSchema) SourceAndTargetVertexIdPair(pr.srcVertexId.innerId, pr.tgtVertexId.innerId).bytes + else VertexId.toSourceVertexId(pr.srcVertexId).bytes + + val row = Bytes.add(rowFirstBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes) + + val qualifier = + if (isTallSchema) Array.empty[Byte] + else pr.tgtIdBytes + + val metaPropsBytes = HBaseSerializable.propsToKeyValuesWithTs(allProps.toSeq) + + def statusCodeWithOp(statusCode: Byte, op: Byte): Array[Byte] = { + val byte = (((statusCode << 4) | op).toByte) + Array.fill(1)(byte.toByte) } + + val value = Bytes.add(statusCodeWithOp(statusCode, op), metaPropsBytes) + + Seq( + SKeyValue(table, row, cf, qualifier, value, ts, op, durability = true) + ) + } + + def walToSKeyValues(walLog: WalLog, + schema: SchemaManager, + tallSchemaVersions: Set[String]): Iterable[SKeyValue] = { + walToIndexEdgeKeyValue(walLog, schema, tallSchemaVersions) ++ + walToSnapshotEdgeKeyValue(walLog, schema, tallSchemaVersions) } def toSKeyValues(s2: S2Graph, element: GraphElement, autoEdgeCreate: Boolean = false): Seq[SKeyValue] = { diff --git a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala index 43148266..665b88e0 100644 --- a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala +++ b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala @@ -1,9 +1,56 @@ package org.apache.s2graph.s2jobs.wal.utils +import org.apache.hadoop.hbase.CellUtil +import org.apache.hadoop.hbase.KeyValue.Type +import org.apache.hadoop.hbase.client.Result +import org.apache.s2graph.core.schema._ +import org.apache.s2graph.s2jobs.wal.{SchemaManager, WalLog} import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers} class DeserializeUtilTest extends FunSuite with Matchers with BeforeAndAfterAll { + def initTestSchemaManager(serviceName: String, + labelName: String) = { + val service = Service(Option(1), serviceName, "token", "cluster", "test", 0, None) + val serviceLs = Seq(service) + val serviceColumn = ServiceColumn(Option(1), service.id.get, "user", "string", "v3", None) + val serviceColumnLs = Seq(serviceColumn) + val columnMeta = ColumnMeta(Option(1), serviceColumn.id.get, "age", 1, "integer", "-1") + val columnMetaLs = Seq(columnMeta) + val label = Label(Option(1), labelName, + service.id.get, serviceColumn.columnName, serviceColumn.columnType, + service.id.get, serviceColumn.columnName, serviceColumn.columnType, + true, service.serviceName, service.id.get, "strong", + "test", None, "v3", false, "gz", None + ) + val labelLs = Seq(label) + val labelMeta = LabelMeta(Option(1), label.id.get, "score", 1, "0.0", "double") + val labelMetaLs = Seq(labelMeta) + val labelIndex = LabelIndex(Option(1), label.id.get, "pk", 1, Seq(labelMeta.seq), "", None, None) + val labelIndexLs = Seq(labelIndex) + + SchemaManager(serviceLs, + serviceColumnLs, + columnMetaLs, + labelLs, + labelIndexLs, + labelMetaLs + ) + } + test("test serialize deserialize") { + val tallSchemaVersions = Set("v4") + val schema = initTestSchemaManager("s1", "l1") + val walLog = + WalLog(10L,"insert","e","a","x","s1","l1","""{"age": 20, "gender": "M"}""") + + val kvs = SerializeUtil.walToIndexEdgeKeyValue(walLog, schema, tallSchemaVersions) + + val cells = kvs.map { kv => + CellUtil.createCell(kv.row, kv.cf, kv.qualifier, kv.timestamp, Type.Put.getCode, kv.value) + } + val result = Result.create(cells.toArray) + val wals = DeserializeUtil.indexEdgeResultToWals(result, schema, tallSchemaVersions) + wals.foreach(println) } } From 420c0bce3011fc89d4ad08a58166470681d308d7 Mon Sep 17 00:00:00 2001 From: DO YUNG YOON Date: Mon, 31 Dec 2018 12:09:23 +0900 Subject: [PATCH 09/13] bug fix on wide/tall schema on Vertex/SerializeUtil. --- .../apache/s2graph/s2jobs/task/Source.scala | 72 +++++------- .../s2graph/s2jobs/wal/SchemaManager.scala | 23 ++-- .../apache/s2graph/s2jobs/wal/WalLog.scala | 24 +++- .../s2jobs/wal/utils/DeserializeUtil.scala | 30 +++-- .../s2jobs/wal/utils/SerializeUtil.scala | 106 ++++++++++++++---- .../spark/sql/streaming/S2SourceConfigs.scala | 1 + .../wal/utils/DeserializeUtilTest.scala | 85 ++++++++++++-- 7 files changed, 246 insertions(+), 95 deletions(-) diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala index 381c94df..40cdade6 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala @@ -19,32 +19,25 @@ package org.apache.s2graph.s2jobs.task -import org.apache.hadoop.hbase.Cell -import org.apache.hadoop.hbase.client.Result -import org.apache.s2graph.core.storage.SKeyValue import org.apache.s2graph.core.types.HBaseType import org.apache.s2graph.core.{JSONParser, Management} -import org.apache.s2graph.s2jobs.Schema -import org.apache.s2graph.s2jobs.loader.{HFileGenerator, SparkBulkLoaderTransformer} -import org.apache.s2graph.s2jobs.serde.reader.S2GraphCellReader -import org.apache.s2graph.s2jobs.serde.writer.RowDataFrameWriter +import org.apache.s2graph.s2jobs.loader.HFileGenerator import org.apache.s2graph.s2jobs.wal.utils.{DeserializeUtil, SchemaUtil} -import org.apache.spark.sql.{DataFrame, Encoders, SparkSession} +import org.apache.spark.sql.{DataFrame, SparkSession} import play.api.libs.json.{JsObject, Json} -import scala.reflect.runtime.universe._ - /** * Source * * @param conf */ -abstract class Source(override val conf:TaskConf) extends Task { - def toDF(ss:SparkSession):DataFrame +abstract class Source(override val conf: TaskConf) extends Task { + def toDF(ss: SparkSession): DataFrame } -class KafkaSource(conf:TaskConf) extends Source(conf) { +class KafkaSource(conf: TaskConf) extends Source(conf) { val DEFAULT_FORMAT = "raw" + override def mandatoryOptions: Set[String] = Set("kafka.bootstrap.servers", "subscribe") def repartition(df: DataFrame, defaultParallelism: Int) = { @@ -57,7 +50,7 @@ class KafkaSource(conf:TaskConf) extends Source(conf) { } } - override def toDF(ss:SparkSession):DataFrame = { + override def toDF(ss: SparkSession): DataFrame = { logger.info(s"${LOG_PREFIX} options: ${conf.options}") val format = conf.options.getOrElse("format", "raw") @@ -67,22 +60,22 @@ class KafkaSource(conf:TaskConf) extends Source(conf) { format match { case "raw" => partitionedDF case "json" => parseJsonSchema(ss, partitionedDF) -// case "custom" => parseCustomSchema(df) + // case "custom" => parseCustomSchema(df) case _ => logger.warn(s"${LOG_PREFIX} unsupported format '$format'.. use default schema ") partitionedDF } } - def parseJsonSchema(ss:SparkSession, df:DataFrame):DataFrame = { + def parseJsonSchema(ss: SparkSession, df: DataFrame): DataFrame = { import org.apache.spark.sql.functions.from_json import org.apache.spark.sql.types.DataType import ss.implicits._ val schemaOpt = conf.options.get("schema") schemaOpt match { - case Some(schemaAsJson:String) => - val dataType:DataType = DataType.fromJson(schemaAsJson) + case Some(schemaAsJson: String) => + val dataType: DataType = DataType.fromJson(schemaAsJson) logger.debug(s"${LOG_PREFIX} schema : ${dataType.sql}") df.selectExpr("CAST(value AS STRING)") @@ -96,8 +89,9 @@ class KafkaSource(conf:TaskConf) extends Source(conf) { } } -class FileSource(conf:TaskConf) extends Source(conf) { +class FileSource(conf: TaskConf) extends Source(conf) { val DEFAULT_FORMAT = "parquet" + override def mandatoryOptions: Set[String] = Set("paths") override def toDF(ss: SparkSession): DataFrame = { @@ -126,7 +120,7 @@ class FileSource(conf:TaskConf) extends Source(conf) { } -class HiveSource(conf:TaskConf) extends Source(conf) { +class HiveSource(conf: TaskConf) extends Source(conf) { override def mandatoryOptions: Set[String] = Set("database", "table") override def toDF(ss: SparkSession): DataFrame = { @@ -138,37 +132,33 @@ class HiveSource(conf:TaskConf) extends Source(conf) { } } -case class HResult(table: Array[Byte], - row: Array[Byte], - cf: Array[Byte], - kvs: Seq[PartialKeyValue]) - -object PartialKeyValue { - def apply(cell: Cell): PartialKeyValue = { - PartialKeyValue(cell.getQualifier, cell.getValue, cell.getTimestamp) - } -} -case class PartialKeyValue(qualifier: Array[Byte], - value: Array[Byte], - timestamp: Long) - - class S2GraphSource(conf: TaskConf) extends Source(conf) { + import org.apache.s2graph.spark.sql.streaming.S2SourceConfigs._ - import scala.collection.mutable - import scala.collection.JavaConverters._ override def mandatoryOptions: Set[String] = Set( S2_SOURCE_BULKLOAD_HBASE_ROOT_DIR, S2_SOURCE_BULKLOAD_RESTORE_PATH, S2_SOURCE_BULKLOAD_HBASE_TABLE_NAMES, - "labelNames" + S2_SOURCE_BULKLOAD_LABEL_NAMES ) override def toDF(ss: SparkSession): DataFrame = { + /* + * overwrite HBASE + MetaStorage + LocalCache configuration from given option. + */ val mergedConf = TaskConf.parseHBaseConfigs(conf) ++ TaskConf.parseMetaStoreConfigs(conf) ++ TaskConf.parseLocalCacheConfigs(conf) val config = Management.toConfig(mergedConf) + + /* + * initialize meta storage connection. + * note that we only connect meta storage once at spark driver, + * then build schema manager which is serializable and broadcast it to executors. + * + * schema manager responsible for translation between logical logical representation and physical representation. + * + */ SchemaUtil.init(config) val sc = ss.sparkContext @@ -178,14 +168,14 @@ class S2GraphSource(conf: TaskConf) extends Source(conf) { val tableNames = conf.options(S2_SOURCE_BULKLOAD_HBASE_TABLE_NAMES).split(",") val columnFamily = conf.options.getOrElse(S2_SOURCE_BULKLOAD_HBASE_TABLE_CF, "e") val batchSize = conf.options.getOrElse(S2_SOURCE_BULKLOAD_SCAN_BATCH_SIZE, "1000").toInt - val labelNames = conf.options("labelNames").split(",").toSeq + val labelNames = conf.options(S2_SOURCE_BULKLOAD_LABEL_NAMES).split(",").toSeq val labelMapping = Map.empty[String, String] val buildDegree = if (columnFamily == "v") false else conf.options.getOrElse(S2_SOURCE_BULKLOAD_BUILD_DEGREE, "false").toBoolean -// val elementType = conf.options.getOrElse(S2_SOURCE_ELEMENT_TYPE, "IndexEdge") -// val schema = if (columnFamily == "v") Schema.VertexSchema else Schema.EdgeSchema + // val elementType = conf.options.getOrElse(S2_SOURCE_ELEMENT_TYPE, "IndexEdge") + // val schema = if (columnFamily == "v") Schema.VertexSchema else Schema.EdgeSchema val cells = HFileGenerator.tableSnapshotDump(ss, config, snapshotPath, restorePath, tableNames, columnFamily, batchSize, labelMapping, buildDegree) diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SchemaManager.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SchemaManager.scala index 70508daf..357d1e3e 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SchemaManager.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SchemaManager.scala @@ -28,13 +28,13 @@ class SchemaManager(serviceLs: Seq[Service], columnsInverted.get(columnId).map(_ -> (metas ++ ColumnMeta.reservedMetas)) } -// private val columnMetas = columnMetasMap.map { case (column, metas) => -// val innerMap = metas.map { meta => -// meta.name -> meta -// }.toMap -// -// column.columnName -> innerMap -// } + private val columnMetas = columnMetasMap.map { case (column, metas) => + val innerMap = metas.map { meta => + meta.name -> meta + }.toMap + + column -> innerMap + } private val columnMetasInverted = columnMetasMap.map { case (column, metas) => val innerMap = metas.map { meta => @@ -180,6 +180,15 @@ class SchemaManager(serviceLs: Seq[Service], serviceNameColumnNameColumn(serviceName -> columnName) } + def findColumnMetas(serviceName: String, columnName: String): Map[String, ColumnMeta] = { + val column = findServiceColumn(serviceName, columnName) + findColumnMetas(column) + } + + def findColumnMetas(column: ServiceColumn): Map[String, ColumnMeta] = { + columnMetas(column) + } + def findLabel(labelName: String): Label = { labels(labelName) } diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/WalLog.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/WalLog.scala index 3c23721e..12641c27 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/WalLog.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/WalLog.scala @@ -1,7 +1,7 @@ package org.apache.s2graph.s2jobs.wal import com.google.common.hash.Hashing -import org.apache.s2graph.core.{GraphUtil, JSONParser} +import org.apache.s2graph.core.JSONParser import org.apache.s2graph.s2jobs.wal.process.params.AggregateParam import org.apache.s2graph.s2jobs.wal.transformer.Transformer import org.apache.s2graph.s2jobs.wal.utils.BoundedPriorityQueue @@ -104,8 +104,8 @@ object WalLogAgg { private def filterPropsInner(walLogs: Seq[WalLog], - transformers: Seq[Transformer], - validFeatureHashKeys: Set[Long]): Seq[WalLog] = { + transformers: Seq[Transformer], + validFeatureHashKeys: Set[Long]): Seq[WalLog] = { walLogs.map { walLog => val fields = walLog.propsJson.fields.filter { case (propKey, propValue) => val filtered = transformers.flatMap { transformer => @@ -119,8 +119,8 @@ object WalLogAgg { } def filterProps(walLogAgg: WalLogAgg, - transformers: Seq[Transformer], - validFeatureHashKeys: Set[Long]) = { + transformers: Seq[Transformer], + validFeatureHashKeys: Set[Long]) = { val filteredVertices = filterPropsInner(walLogAgg.vertices, transformers, validFeatureHashKeys) val filteredEdges = filterPropsInner(walLogAgg.edges, transformers, validFeatureHashKeys) @@ -176,6 +176,20 @@ case class WalLog(timestamp: Long, } } +object WalVertex { + def fromWalLog(walLog: WalLog): WalVertex = { + WalVertex( + walLog.timestamp, + walLog.operation, + walLog.elem, + walLog.from, + walLog.service, + walLog.label, + walLog.props + ) + } +} + case class WalVertex(timestamp: Long, operation: String, elem: String, diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala index 074923a1..338312fe 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala @@ -196,12 +196,13 @@ object DeserializeUtil { val countVal = StorageDeserializable.bytesToLong(value, 0) Array(LabelMeta.count -> InnerVal.withLong(countVal, schemaVer)) } else { - val (props, endAt) = bytesToKeyValues(value, 0, value.length, schemaVer, schema.findLabelMetas(labelId)) + val (props, _) = bytesToKeyValues(value, 0, value.length, schemaVer, schema.findLabelMetas(labelId)) props } } - def toSnapshotRowV3Parsed(row: Array[Byte]): Try[SnapshotRowV3Parsed] = Try { + def toSnapshotRowV3Parsed(row: Array[Byte], + qualifier: Array[Byte]): Try[SnapshotRowV3Parsed] = Try { val labelWithDirByteLen = 4 val labelIndexSeqWithIsInvertedByteLen = 1 @@ -209,7 +210,7 @@ object DeserializeUtil { SourceVertexId.fromBytes(row, 0, row.length, HBaseType.DEFAULT_VERSION) val isTallSchema = - srcIdLen + labelWithDirByteLen + labelIndexSeqWithIsInvertedByteLen != row.length + (srcIdLen + labelWithDirByteLen + labelIndexSeqWithIsInvertedByteLen) != row.length val (tgtVertexId, pos) = if (isTallSchema) { @@ -217,7 +218,9 @@ object DeserializeUtil { (TargetVertexId(ServiceColumn.Default, tgtId), srcIdLen + tgtBytesLen) } else { - (TargetVertexId(ServiceColumn.Default, srcVertexId.innerId), srcIdLen) + val (tgtVertexId, _) = TargetVertexId.fromBytes(qualifier, 0, qualifier.length, HBaseType.DEFAULT_VERSION) + + (tgtVertexId, srcIdLen) } val labelWithDir = LabelWithDirection(Bytes.toInt(row, pos, labelWithDirByteLen)) @@ -229,8 +232,8 @@ object DeserializeUtil { def deserializeIndexEdgeCell(cell: Cell, row: Array[Byte], rowV3Parsed: RowV3Parsed, - tallSchemaVersions: Set[String], - schema: SchemaManager): Option[WalLog] = { + schema: SchemaManager, + tallSchemaVersions: Set[String]): Option[WalLog] = { val labelWithDir = rowV3Parsed.labelWithDir val labelId = labelWithDir.labelId val labelIdxSeq = rowV3Parsed.labelIdxSeq @@ -306,7 +309,7 @@ object DeserializeUtil { if (inValidRow) Nil else { rawCells.flatMap { cell => - deserializeIndexEdgeCell(cell, row, rowV3Parsed, tallSchemaVersions, schema) + deserializeIndexEdgeCell(cell, row, rowV3Parsed, schema, tallSchemaVersions) } } } @@ -322,8 +325,9 @@ object DeserializeUtil { else { val head = rawCells.head val row = head.getRow + val qualifier = head.getQualifier - toSnapshotRowV3Parsed(row) match { + toSnapshotRowV3Parsed(row, qualifier) match { case Success(v) => val SnapshotRowV3Parsed(srcVertexId, tgtVertexId, labelWithDir, _, isInverted) = v @@ -378,9 +382,10 @@ object DeserializeUtil { val row = head.getRow val version = HBaseType.DEFAULT_VERSION - val (vertexInnerId, colId, _) = VertexId.fromBytesRaw(row, 0, row.length, version) + val (vertexInnerId, colId, len) = VertexId.fromBytesRaw(row, 0, row.length, version) val column = schema.findServiceColumn(colId) + val columnMetaSeqExistInRow = len + 1 == row.length var maxTs = Long.MinValue val propsMap = mutable.Map.empty[ColumnMeta, InnerValLike] @@ -390,8 +395,11 @@ object DeserializeUtil { val qualifier = cell.getQualifier val propKey = - if (qualifier.length == 1) qualifier.head.toInt - else bytesToInt(qualifier, 0) + if (columnMetaSeqExistInRow) row.last + else { + if (qualifier.length == 1) qualifier.head.toInt + else bytesToInt(qualifier, 0) + } val ts = cell.getTimestamp diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala index 974847bc..a950e9f3 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala @@ -2,11 +2,12 @@ package org.apache.s2graph.s2jobs.wal.utils import org.apache.hadoop.hbase.util.Bytes import org.apache.s2graph.core._ -import org.apache.s2graph.core.schema.{Label, LabelIndex, LabelMeta, ServiceColumn} +import org.apache.s2graph.core.schema._ import org.apache.s2graph.core.storage.SKeyValue import org.apache.s2graph.core.storage.serde.Serializable import org.apache.s2graph.core.types._ -import org.apache.s2graph.s2jobs.wal.{SchemaManager, WalLog} +import org.apache.s2graph.s2jobs.wal.{SchemaManager, WalLog, WalVertex} +import play.api.libs.json.{JsObject, Json} object SerializeUtil { @@ -25,11 +26,11 @@ object SerializeUtil { snapshotEdgeKeyValues ++ indexEdgeKeyValues } - def buildIndexInnerProps(walLog: WalLog, - innerProps: Map[LabelMeta, InnerValLike], - defaultProps: Map[LabelMeta, InnerValLike], - label: Label, - labelMetasInOrder: Array[LabelMeta]): Array[(LabelMeta, InnerValLike)] = { + def buildEdgeIndexInnerProps(walLog: WalLog, + innerProps: Map[LabelMeta, InnerValLike], + defaultProps: Map[LabelMeta, InnerValLike], + label: Label, + labelMetasInOrder: Array[LabelMeta]): Array[(LabelMeta, InnerValLike)] = { labelMetasInOrder.map { labelMeta => innerProps.get(labelMeta) match { case None => @@ -42,10 +43,10 @@ object SerializeUtil { } } - def buildInnerDefaultProps(walLog: WalLog, - label: Label, - srcColumn: ServiceColumn, - tgtColumn: ServiceColumn): Map[LabelMeta, InnerValLike] = { + def buildEdgeInnerDefaultProps(walLog: WalLog, + label: Label, + srcColumn: ServiceColumn, + tgtColumn: ServiceColumn): Map[LabelMeta, InnerValLike] = { Map( LabelMeta.timestamp -> InnerVal.withLong(walLog.timestamp, label.schemaVersion), LabelMeta.from -> JSONParser.toInnerVal(walLog.from, srcColumn.columnType, srcColumn.schemaVersion), @@ -53,19 +54,42 @@ object SerializeUtil { ) } - def buildInnerProps(walLog: WalLog, - label: Label, - schema: SchemaManager): Map[LabelMeta, InnerValLike] = { + def buildEdgeInnerProps(walLog: WalLog, + label: Label, + schema: SchemaManager): Map[LabelMeta, InnerValLike] = { + val ts = walLog.timestamp + val schemaVer = label.schemaVersion + val tsInnerVal = InnerVal.withLong(ts, schemaVer) + val labelMetas = schema.findLabelMetas(walLog.label) val props = walLog.propsJson.fields.flatMap { case (key, jsValue) => labelMetas.get(key).flatMap { labelMeta => - JSONParser.jsValueToInnerVal(jsValue, labelMeta.dataType, label.schemaVersion).map { innerVal => + JSONParser.jsValueToInnerVal(jsValue, labelMeta.dataType, schemaVer).map { innerVal => labelMeta -> innerVal } } }.toMap - props ++ Map(LabelMeta.timestamp -> InnerVal.withLong(walLog.timestamp, label.schemaVersion)) + props ++ Map(LabelMeta.timestamp -> tsInnerVal) + } + + def buildVertexInnerProps(walVertex: WalVertex, + column: ServiceColumn, + schema: SchemaManager): Map[ColumnMeta, InnerValLike] = { + val ts = walVertex.timestamp + val schemaVer = column.schemaVersion + val tsInnerVal = InnerVal.withLong(ts, schemaVer) + + val columnMetas = schema.findColumnMetas(column) + val props = Json.parse(walVertex.props).as[JsObject].fields.flatMap { case (key, jsValue) => + columnMetas.get(key).flatMap { columnMeta => + JSONParser.jsValueToInnerVal(jsValue, columnMeta.dataType, schemaVer).map { innerVal => + columnMeta -> innerVal + } + } + }.toMap + + props ++ Map(ColumnMeta.timestamp -> tsInnerVal, ColumnMeta.lastModifiedAtColumn -> tsInnerVal) } case class PartialResult(label: Label, @@ -101,8 +125,8 @@ object SerializeUtil { val tgtInnerVal = JSONParser.toInnerVal(walLog.to, tgtColumn.columnType, tgtColumn.schemaVersion) val tgtVertexId = VertexId(tgtColumn, tgtInnerVal) - val allProps = buildInnerProps(walLog, label, schema) - val defaultProps = buildInnerDefaultProps(walLog, label, srcColumn, tgtColumn) + val allProps = buildEdgeInnerProps(walLog, label, schema) + val defaultProps = buildEdgeInnerDefaultProps(walLog, label, srcColumn, tgtColumn) val op = GraphUtil.toOp(walLog.operation).getOrElse(throw new IllegalArgumentException(s"${walLog.operation} operation is not supported.")) @@ -128,7 +152,7 @@ object SerializeUtil { schema.findLabelIndices(label).map { case (indexName, index) => val labelMetasInOrder = schema.findLabelIndexLabelMetas(label.label, indexName) - val orderedProps = buildIndexInnerProps(walLog, allProps, defaultProps, label, labelMetasInOrder) + val orderedProps = buildEdgeIndexInnerProps(walLog, allProps, defaultProps, label, labelMetasInOrder) val idxPropsMap = orderedProps.toMap val idxPropsBytes = propsToBytes(orderedProps) val metaProps = allProps.filterKeys(m => !idxPropsMap.contains(m)) @@ -230,6 +254,50 @@ object SerializeUtil { walToSnapshotEdgeKeyValue(walLog, schema, tallSchemaVersions) } + def walVertexToSKeyValue(v: WalVertex, + schema: SchemaManager, + tallSchemaVersions: Set[String]): Iterable[SKeyValue] = { + val column = schema.findServiceColumn(v.service, v.column) + val service = schema.findColumnService(column.id.get) + val schemaVer = column.schemaVersion + + val innerVal = JSONParser.toInnerVal(v.id, column.columnType, column.schemaVersion) + val vertexId = VertexId(column, innerVal) + val row = vertexId.bytes + + val props = buildVertexInnerProps(v, column, schema) + //TODO: skip belongLabelIds + + val table = service.hTableName.getBytes("UTF-8") + val cf = Serializable.vertexCf + val isTallSchema = tallSchemaVersions(schemaVer) + + props.map { case (columnMeta, innerVal) => + val rowBytes = + if (isTallSchema) Bytes.add(row, Array.fill(1)(columnMeta.seq)) + else row + + val qualifier = + if (isTallSchema) Array.empty[Byte] + else intToBytes(columnMeta.seq) + + val value = innerVal.bytes + + SKeyValue(table, rowBytes, cf, qualifier, value, v.timestamp) + } + } + + // main public interface. + def toSKeyValues(walLog: WalLog, + schema: SchemaManager, + tallSchemaVersions: Set[String]): Iterable[SKeyValue] = { + walLog.elem match { + case "vertex" | "v" => walVertexToSKeyValue(WalVertex.fromWalLog(walLog), schema, tallSchemaVersions) + case "edge" | "e" => walToSKeyValues(walLog, schema, tallSchemaVersions) + case _ => throw new IllegalArgumentException(s"$walLog ${walLog.elem} is not supported.") + } + } + def toSKeyValues(s2: S2Graph, element: GraphElement, autoEdgeCreate: Boolean = false): Seq[SKeyValue] = { if (element.isInstanceOf[S2Edge]) { val edge = element.asInstanceOf[S2Edge] diff --git a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SourceConfigs.scala b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SourceConfigs.scala index 8e205e3f..d5b27bb5 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SourceConfigs.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SourceConfigs.scala @@ -33,6 +33,7 @@ object S2SourceConfigs { val S2_SOURCE_BULKLOAD_HBASE_TABLE_NAMES = s"$S2_SOURCE_BULKLOAD_PREFIX.hbase.table.names" val S2_SOURCE_BULKLOAD_HBASE_TABLE_CF = s"$S2_SOURCE_BULKLOAD_PREFIX.hbase.table.cf" val S2_SOURCE_BULKLOAD_SCAN_BATCH_SIZE = s"$S2_SOURCE_BULKLOAD_PREFIX.scan.batch.size" + val S2_SOURCE_BULKLOAD_LABEL_NAMES = s"$S2_SOURCE_BULKLOAD_PREFIX.label.names" // BULKLOAD val S2_SOURCE_BULKLOAD_BUILD_DEGREE = s"$S2_SOURCE_BULKLOAD_PREFIX.build.degree" diff --git a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala index 665b88e0..115f4893 100644 --- a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala +++ b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala @@ -4,15 +4,19 @@ import org.apache.hadoop.hbase.CellUtil import org.apache.hadoop.hbase.KeyValue.Type import org.apache.hadoop.hbase.client.Result import org.apache.s2graph.core.schema._ -import org.apache.s2graph.s2jobs.wal.{SchemaManager, WalLog} +import org.apache.s2graph.core.storage.SKeyValue +import org.apache.s2graph.s2jobs.wal.{SchemaManager, WalLog, WalVertex} import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers} +import play.api.libs.json.{JsObject, JsValue, Json} + +class DeserializeUtilTest extends FunSuite with Matchers with BeforeAndAfterAll { -class DeserializeUtilTest extends FunSuite with Matchers with BeforeAndAfterAll { def initTestSchemaManager(serviceName: String, - labelName: String) = { + columnName: String, + labelName: String) = { val service = Service(Option(1), serviceName, "token", "cluster", "test", 0, None) val serviceLs = Seq(service) - val serviceColumn = ServiceColumn(Option(1), service.id.get, "user", "string", "v3", None) + val serviceColumn = ServiceColumn(Option(1), service.id.get, columnName, "string", "v3", None) val serviceColumnLs = Seq(serviceColumn) val columnMeta = ColumnMeta(Option(1), serviceColumn.id.get, "age", 1, "integer", "-1") val columnMetaLs = Seq(columnMeta) @@ -37,20 +41,77 @@ class DeserializeUtilTest extends FunSuite with Matchers with BeforeAndAfterAll ) } - test("test serialize deserialize") { - val tallSchemaVersions = Set("v4") - val schema = initTestSchemaManager("s1", "l1") - val walLog = - WalLog(10L,"insert","e","a","x","s1","l1","""{"age": 20, "gender": "M"}""") + def checkEqual(walVertex: WalVertex, deserializedWalVertex: WalVertex): Boolean = { + val msg = Seq("=" * 100, walVertex, deserializedWalVertex, "=" * 100).mkString("\n") + println(msg) - val kvs = SerializeUtil.walToIndexEdgeKeyValue(walLog, schema, tallSchemaVersions) + val expectedProps = Json.parse(walVertex.props).as[JsObject] + val realProps = Json.parse(deserializedWalVertex.props).as[JsObject] + val isPropsSame = expectedProps.fieldSet.forall { case (k, expectedVal) => + (realProps \ k).asOpt[JsValue].map { realVal => expectedVal == realVal }.getOrElse(false) + } + + val isWalLogSame = walVertex.copy(props = "{}") == deserializedWalVertex.copy(props = "{}") + + isWalLogSame && isPropsSame + } + + def checkEqual(walLog: WalLog, deserializedWalLog: WalLog): Boolean = { + val msg = Seq("=" * 100, walLog, deserializedWalLog, "=" * 100).mkString("\n") + println(msg) + val expectedProps = Json.parse(walLog.props).as[JsObject] + val realProps = Json.parse(deserializedWalLog.props).as[JsObject] + val isPropsSame = expectedProps.fieldSet.forall { case (k, expectedVal) => + (realProps \ k).asOpt[JsValue].map { realVal => expectedVal == realVal }.getOrElse(false) + } + + val isWalLogSame = walLog.copy(props = "{}") == deserializedWalLog.copy(props = "{}") + + isWalLogSame && isPropsSame + } + + def createResult(kvs: Iterable[SKeyValue]): Result = { val cells = kvs.map { kv => CellUtil.createCell(kv.row, kv.cf, kv.qualifier, kv.timestamp, Type.Put.getCode, kv.value) } - val result = Result.create(cells.toArray) + Result.create(cells.toArray) + } + + val schema = initTestSchemaManager("s1", "user", "l1") + val tallSchemaVersions = Set("v4") + val walLog = + WalLog(10L, "insert", "edge", "a", "x", "s1", "l1","""{"score": 20}""") + + val walVertex = + WalVertex(10L, "insert", "vertex", "v1", "s1", "user", """{"age": 20}""") + + test("test index edge serialize/deserialize") { + val kvs = SerializeUtil.walToIndexEdgeKeyValue(walLog, schema, tallSchemaVersions) + val result = createResult(kvs) val wals = DeserializeUtil.indexEdgeResultToWals(result, schema, tallSchemaVersions) - wals.foreach(println) + + wals.size shouldBe 1 + checkEqual(walLog, wals.head) shouldBe true + } + + test("test snapshot edge serialize/deserialize") { + val kvs = SerializeUtil.walToSnapshotEdgeKeyValue(walLog, schema, tallSchemaVersions) + val result = createResult(kvs) + val wals = DeserializeUtil.snapshotEdgeResultToWals(result, schema, tallSchemaVersions) + + wals.size shouldBe 1 + checkEqual(walLog, wals.head) shouldBe true + } + + test("test vertex serialize/deserialize") { + val kvs = SerializeUtil.walVertexToSKeyValue(walVertex, schema, tallSchemaVersions) + val result = createResult(kvs) + val wals = DeserializeUtil.vertexResultToWals(result, schema) + + wals.size shouldBe 1 + checkEqual(walVertex, wals.head) shouldBe true + } } From 48b1594417f476256383811405bae8728f3e1780 Mon Sep 17 00:00:00 2001 From: DO YUNG YOON Date: Mon, 31 Dec 2018 12:23:52 +0900 Subject: [PATCH 10/13] Need to pass right spark sql schema on createDataFrame. --- .../org/apache/s2graph/s2jobs/task/Sink.scala | 5 ++-- .../apache/s2graph/s2jobs/task/Source.scala | 10 +++++-- .../s2graph/s2jobs/wal/utils/SchemaUtil.scala | 28 ++++++++----------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Sink.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Sink.scala index dd6f41b7..77171b07 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Sink.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Sink.scala @@ -25,6 +25,7 @@ import org.apache.s2graph.s2jobs.S2GraphHelper import org.apache.s2graph.s2jobs.loader.{HFileGenerator, SparkBulkLoaderTransformer} import org.apache.s2graph.s2jobs.serde.reader.RowBulkFormatReader import org.apache.s2graph.s2jobs.serde.writer.KeyValueWriter +import org.apache.s2graph.s2jobs.wal.utils.SchemaUtil import org.apache.spark.sql._ import org.apache.spark.sql.streaming.{DataStreamWriter, OutputMode, Trigger} import org.elasticsearch.spark.sql.EsSparkSQL @@ -247,10 +248,10 @@ class S2GraphSink(queryName: String, conf: TaskConf) extends Sink(queryName, con implicit val reader = new RowBulkFormatReader implicit val writer = new KeyValueWriter(autoEdgeCreate, skipError) - val kvs = transformer.transform(input) + val kvs = transformer.transform(input).flatMap(ls => ls) HFileGenerator.generateHFile(df.sparkSession.sparkContext, graphConfig, - kvs.flatMap(ls => ls), hbaseConfig, tableName, + kvs, hbaseConfig, tableName, numRegions, outputPath, incrementalLoad, compressionAlgorithm ) diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala index 40cdade6..94088c84 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala @@ -174,7 +174,7 @@ class S2GraphSource(conf: TaskConf) extends Source(conf) { val buildDegree = if (columnFamily == "v") false else conf.options.getOrElse(S2_SOURCE_BULKLOAD_BUILD_DEGREE, "false").toBoolean - // val elementType = conf.options.getOrElse(S2_SOURCE_ELEMENT_TYPE, "IndexEdge") + val elementType = conf.options.getOrElse(S2_SOURCE_ELEMENT_TYPE, "IndexEdge") // val schema = if (columnFamily == "v") Schema.VertexSchema else Schema.EdgeSchema val cells = HFileGenerator.tableSnapshotDump(ss, config, snapshotPath, @@ -189,9 +189,13 @@ class S2GraphSource(conf: TaskConf) extends Source(conf) { val schema = schemaBCast.value iter.flatMap { case (_, result) => - DeserializeUtil.indexEdgeResultToWals(result, schema, tallSchemaVersions, tgtDirection) + elementType.toLowerCase match { + case "indexedge" => DeserializeUtil.indexEdgeResultToWals(result, schema, tallSchemaVersions, tgtDirection) + case "snapshotedge" => DeserializeUtil.snapshotEdgeResultToWals(result, schema, tallSchemaVersions) +// case "vertex" => DeserializeUtil.vertexResultToWals(result, schema) + case _ => throw new IllegalArgumentException(s"$elementType is not supported.") + } } - } ss.createDataFrame(results) diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala index c65e6e6c..19425a19 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala @@ -64,28 +64,16 @@ object SchemaUtil { } } - def buildVertexDeserializeSchema(serviceNameColumnNames: Map[String, Seq[String]]): SchemaManager = { - val serviceLs = serviceNameColumnNames.keys.toSeq.map { serviceName => + def buildSchemaManager(serviceNameColumnNames: Map[String, Seq[String]], + labelNames: Seq[String]): SchemaManager = { + val _serviceLs = serviceNameColumnNames.keys.toSeq.map { serviceName => Service.findByName(serviceName).getOrElse(throw new IllegalArgumentException(s"$serviceName not found.")) } - - val serviceColumnLs = serviceLs.flatMap { service => - ServiceColumn.findByServiceId(service.id.get) - } - - val columnMetaLs = serviceColumnLs.flatMap { column => - column.metas - } - - SchemaManager(serviceLs, serviceColumnLs, columnMetaLs, Nil, Nil, Nil) - } - - def buildEdgeDeserializeSchema(labelNames: Seq[String]): SchemaManager = { val labelLs = labelNames.map { labelName => Label.findByName(labelName).getOrElse(throw new IllegalArgumentException(s"$labelName not exist.")) } - val serviceLs = labelLs.flatMap { label => + val serviceLs = _serviceLs ++ labelLs.flatMap { label => Seq(label.srcService, label.tgtService) }.distinct @@ -106,4 +94,12 @@ object SchemaUtil { SchemaManager(serviceLs, serviceColumnLs, columnMetaLs, labelLs, labelIndexLs, labelMetaLs) } + + def buildVertexDeserializeSchema(serviceNameColumnNames: Map[String, Seq[String]]): SchemaManager = { + buildSchemaManager(serviceNameColumnNames, Nil) + } + + def buildEdgeDeserializeSchema(labelNames: Seq[String]): SchemaManager = { + buildSchemaManager(Map.empty, labelNames) + } } From bf5cfe61ba714cf5266183d48074a3a954c76536 Mon Sep 17 00:00:00 2001 From: DO YUNG YOON Date: Mon, 31 Dec 2018 12:35:41 +0900 Subject: [PATCH 11/13] support vertex in S2GraphSource. --- .../org/apache/s2graph/s2jobs/task/Source.scala | 14 ++++++++------ .../s2graph/s2jobs/wal/utils/DeserializeUtil.scala | 12 ++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala index 94088c84..45bedb0d 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala @@ -21,6 +21,7 @@ package org.apache.s2graph.s2jobs.task import org.apache.s2graph.core.types.HBaseType import org.apache.s2graph.core.{JSONParser, Management} +import org.apache.s2graph.s2jobs.Schema import org.apache.s2graph.s2jobs.loader.HFileGenerator import org.apache.s2graph.s2jobs.wal.utils.{DeserializeUtil, SchemaUtil} import org.apache.spark.sql.{DataFrame, SparkSession} @@ -174,8 +175,9 @@ class S2GraphSource(conf: TaskConf) extends Source(conf) { val buildDegree = if (columnFamily == "v") false else conf.options.getOrElse(S2_SOURCE_BULKLOAD_BUILD_DEGREE, "false").toBoolean - val elementType = conf.options.getOrElse(S2_SOURCE_ELEMENT_TYPE, "IndexEdge") - // val schema = if (columnFamily == "v") Schema.VertexSchema else Schema.EdgeSchema + val elementType = conf.options.getOrElse(S2_SOURCE_ELEMENT_TYPE, "IndexEdge").toLowerCase + + val resultSchema = if (columnFamily == "v") Schema.VertexSchema else Schema.EdgeSchema val cells = HFileGenerator.tableSnapshotDump(ss, config, snapshotPath, restorePath, tableNames, columnFamily, batchSize, labelMapping, buildDegree) @@ -190,14 +192,14 @@ class S2GraphSource(conf: TaskConf) extends Source(conf) { iter.flatMap { case (_, result) => elementType.toLowerCase match { - case "indexedge" => DeserializeUtil.indexEdgeResultToWals(result, schema, tallSchemaVersions, tgtDirection) - case "snapshotedge" => DeserializeUtil.snapshotEdgeResultToWals(result, schema, tallSchemaVersions) -// case "vertex" => DeserializeUtil.vertexResultToWals(result, schema) + case "indexedge" => DeserializeUtil.indexEdgeResultToWals(result, schema, tallSchemaVersions, tgtDirection).map(DeserializeUtil.walLogToRow) + case "snapshotedge" => DeserializeUtil.snapshotEdgeResultToWals(result, schema, tallSchemaVersions).map(DeserializeUtil.walLogToRow) + case "vertex" => DeserializeUtil.vertexResultToWals(result, schema).map(DeserializeUtil.walVertexToRow) case _ => throw new IllegalArgumentException(s"$elementType is not supported.") } } } - ss.createDataFrame(results) + ss.createDataFrame(results, resultSchema) } } \ No newline at end of file diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala index 338312fe..74431883 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala @@ -369,6 +369,18 @@ object DeserializeUtil { } } + def walLogToRow(walLog: WalLog): Row = { + Row.fromSeq( + Seq(walLog.timestamp, walLog.operation, walLog.elem, walLog.from, walLog.to, walLog.service, walLog.label, walLog.props) + ) + } + + def walVertexToRow(walVertex: WalVertex): Row = { + Row.fromSeq( + Seq(walVertex.timestamp, walVertex.operation, walVertex.elem, walVertex.id, walVertex.service, walVertex.column, walVertex.props) + ) + } + def vertexResultToWals(result: Result, schema: SchemaManager, bytesToInt: (Array[Byte], Int) => Int = bytesToInt): Seq[WalVertex] = { From c00b9d0ac49f84bb1d17568f52d8a5de77ba37a4 Mon Sep 17 00:00:00 2001 From: DO YUNG YOON Date: Mon, 31 Dec 2018 14:13:35 +0900 Subject: [PATCH 12/13] refactor S2GraphSink bulkload to use SchemaManager to build RDD[KeyValue]. --- .../org/apache/s2graph/s2jobs/task/Sink.scala | 77 ++++++++++++++----- .../s2graph/s2jobs/wal/utils/SchemaUtil.scala | 22 ++++-- .../s2jobs/wal/utils/SerializeUtil.scala | 6 ++ 3 files changed, 78 insertions(+), 27 deletions(-) diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Sink.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Sink.scala index 77171b07..cab97606 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Sink.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Sink.scala @@ -20,12 +20,15 @@ package org.apache.s2graph.s2jobs.task import com.typesafe.config.{Config, ConfigFactory, ConfigRenderOptions} -import org.apache.s2graph.core.GraphUtil +import org.apache.hadoop.hbase.KeyValue +import org.apache.s2graph.core.types.HBaseType +import org.apache.s2graph.core.{GraphUtil, Management} import org.apache.s2graph.s2jobs.S2GraphHelper import org.apache.s2graph.s2jobs.loader.{HFileGenerator, SparkBulkLoaderTransformer} import org.apache.s2graph.s2jobs.serde.reader.RowBulkFormatReader import org.apache.s2graph.s2jobs.serde.writer.KeyValueWriter -import org.apache.s2graph.s2jobs.wal.utils.SchemaUtil +import org.apache.s2graph.s2jobs.wal.WalLog +import org.apache.s2graph.s2jobs.wal.utils.{SchemaUtil, SerializeUtil} import org.apache.spark.sql._ import org.apache.spark.sql.streaming.{DataStreamWriter, OutputMode, Trigger} import org.elasticsearch.spark.sql.EsSparkSQL @@ -220,38 +223,70 @@ class S2GraphSink(queryName: String, conf: TaskConf) extends Sink(queryName, con override val FORMAT: String = "org.apache.s2graph.spark.sql.streaming.S2SinkProvider" private def writeBatchBulkload(df: DataFrame, runLoadIncrementalHFiles: Boolean = true): Unit = { - val mergedOptions = conf.options ++ TaskConf.parseLocalCacheConfigs(conf) - val graphConfig: Config = ConfigFactory.parseMap(mergedOptions).withFallback(ConfigFactory.load()) + /* + * overwrite HBASE + MetaStorage + LocalCache configuration from given option. + */ + val mergedConf = TaskConf.parseHBaseConfigs(conf) ++ TaskConf.parseMetaStoreConfigs(conf) ++ + TaskConf.parseLocalCacheConfigs(conf) + val config = Management.toConfig(mergedConf) + + /* + * initialize meta storage connection. + * note that we only connect meta storage once at spark driver, + * then build schema manager which is serializable and broadcast it to executors. + * + * schema manager responsible for translation between logical logical representation and physical representation. + * + */ + SchemaUtil.init(config) + val ss = df.sparkSession + val sc = ss.sparkContext // required for bulkload - val labelMapping = getConfigStringOpt(graphConfig, S2_SINK_BULKLOAD_LABEL_MAPPING).map(GraphUtil.toLabelMapping).getOrElse(Map.empty) - val buildDegree = getConfigStringOpt(graphConfig, S2_SINK_BULKLOAD_BUILD_DEGREE).map(_.toBoolean).getOrElse(false) - val autoEdgeCreate = getConfigStringOpt(graphConfig, S2_SINK_BULKLOAD_AUTO_EDGE_CREATE).map(_.toBoolean).getOrElse(false) - val skipError = getConfigStringOpt(graphConfig, S2_SINK_SKIP_ERROR).map(_.toBoolean).getOrElse(false) + val labelMapping = getConfigStringOpt(config, S2_SINK_BULKLOAD_LABEL_MAPPING).map(GraphUtil.toLabelMapping).getOrElse(Map.empty) + val buildDegree = getConfigStringOpt(config, S2_SINK_BULKLOAD_BUILD_DEGREE).map(_.toBoolean).getOrElse(false) + val autoEdgeCreate = getConfigStringOpt(config, S2_SINK_BULKLOAD_AUTO_EDGE_CREATE).map(_.toBoolean).getOrElse(false) + val skipError = getConfigStringOpt(config, S2_SINK_SKIP_ERROR).map(_.toBoolean).getOrElse(false) - val zkQuorum = graphConfig.getString(HBaseConfigs.HBASE_ZOOKEEPER_QUORUM) - val tableName = graphConfig.getString(S2_SINK_BULKLOAD_HBASE_TABLE_NAME) + val zkQuorum = config.getString(HBaseConfigs.HBASE_ZOOKEEPER_QUORUM) + val tableName = config.getString(S2_SINK_BULKLOAD_HBASE_TABLE_NAME) - val numRegions = graphConfig.getString(S2_SINK_BULKLOAD_HBASE_NUM_REGIONS).toInt - val outputPath = graphConfig.getString(S2_SINK_BULKLOAD_HBASE_TEMP_DIR) + val numRegions = config.getString(S2_SINK_BULKLOAD_HBASE_NUM_REGIONS).toInt + val outputPath = config.getString(S2_SINK_BULKLOAD_HBASE_TEMP_DIR) // optional. - val incrementalLoad = getConfigStringOpt(graphConfig, S2_SINK_BULKLOAD_HBASE_INCREMENTAL_LOAD).map(_.toBoolean).getOrElse(false) - val compressionAlgorithm = getConfigStringOpt(graphConfig, S2_SINK_BULKLOAD_HBASE_COMPRESSION).getOrElse("lz4") + val incrementalLoad = getConfigStringOpt(config, S2_SINK_BULKLOAD_HBASE_INCREMENTAL_LOAD).map(_.toBoolean).getOrElse(false) + val compressionAlgorithm = getConfigStringOpt(config, S2_SINK_BULKLOAD_HBASE_COMPRESSION).getOrElse("lz4") val hbaseConfig = HFileGenerator.toHBaseConfig(zkQuorum, tableName) - val input = df.rdd + val schema = SchemaUtil.buildSchemaManager(Map.empty, Nil) + val schemaBCast = sc.broadcast(schema) + val tallSchemaVersions = Set(HBaseType.VERSION4) + + implicit val enc = Encoders.kryo[KeyValue] - val transformer = new SparkBulkLoaderTransformer(graphConfig, labelMapping, buildDegree) + val kvs = df.mapPartitions { iter => + val schema = schemaBCast.value - implicit val reader = new RowBulkFormatReader - implicit val writer = new KeyValueWriter(autoEdgeCreate, skipError) + iter.flatMap { row => + val walLog = WalLog.fromRow(row) + + SerializeUtil.walToSKeyValues(walLog, schema, tallSchemaVersions) + .map(SerializeUtil.sKeyValueToKeyValue) + } + } - val kvs = transformer.transform(input).flatMap(ls => ls) +// val input = df +// val transformer = new SparkBulkLoaderTransformer(graphConfig, labelMapping, buildDegree) +// +// implicit val reader = new RowBulkFormatReader +// implicit val writer = new KeyValueWriter(autoEdgeCreate, skipError) +// +// val kvs = transformer.transform(input).flatMap(ls => ls) - HFileGenerator.generateHFile(df.sparkSession.sparkContext, graphConfig, - kvs, hbaseConfig, tableName, + HFileGenerator.generateHFile(df.sparkSession.sparkContext, config, + kvs.rdd, hbaseConfig, tableName, numRegions, outputPath, incrementalLoad, compressionAlgorithm ) diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala index 19425a19..f1f4bbf1 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala @@ -66,12 +66,22 @@ object SchemaUtil { def buildSchemaManager(serviceNameColumnNames: Map[String, Seq[String]], labelNames: Seq[String]): SchemaManager = { - val _serviceLs = serviceNameColumnNames.keys.toSeq.map { serviceName => - Service.findByName(serviceName).getOrElse(throw new IllegalArgumentException(s"$serviceName not found.")) - } - val labelLs = labelNames.map { labelName => - Label.findByName(labelName).getOrElse(throw new IllegalArgumentException(s"$labelName not exist.")) - } + + val _serviceLs = + if (serviceNameColumnNames.isEmpty) Service.findAll() + else { + serviceNameColumnNames.keys.toSeq.map { serviceName => + Service.findByName(serviceName).getOrElse(throw new IllegalArgumentException(s"$serviceName not found.")) + } + } + + val labelLs = + if (labelNames.isEmpty) Label.findAll() + else { + labelNames.map { labelName => + Label.findByName(labelName).getOrElse(throw new IllegalArgumentException(s"$labelName not exist.")) + } + } val serviceLs = _serviceLs ++ labelLs.flatMap { label => Seq(label.srcService, label.tgtService) diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala index a950e9f3..2f54512e 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala @@ -1,5 +1,7 @@ package org.apache.s2graph.s2jobs.wal.utils +import org.apache.hadoop.hbase.KeyValue +import org.apache.hadoop.hbase.KeyValue.Type import org.apache.hadoop.hbase.util.Bytes import org.apache.s2graph.core._ import org.apache.s2graph.core.schema._ @@ -309,4 +311,8 @@ object SerializeUtil { Nil } } + + def sKeyValueToKeyValue(skv: SKeyValue): KeyValue = { + new KeyValue(skv.row, skv.cf, skv.qualifier, skv.timestamp, Type.Put, skv.value) + } } From a27b9d6cf81f0461c6dd31e5f519ccb022f442ad Mon Sep 17 00:00:00 2001 From: DO YUNG YOON Date: Mon, 31 Dec 2018 19:19:39 +0900 Subject: [PATCH 13/13] extract SchemaManager as trait and added MemeorySchemaManager on s2jobs. --- .../s2graph/core/schema/SchemaManager.scala | 59 ++++++++++++++ .../s2graph/s2jobs/wal/ColumnSchema.scala | 48 ------------ .../s2graph/s2jobs/wal/LabelSchema.scala | 10 --- ...anager.scala => MemorySchemaManager.scala} | 16 ++-- .../s2jobs/wal/utils/DeserializeUtil.scala | 2 +- .../s2graph/s2jobs/wal/utils/SchemaUtil.scala | 4 +- .../s2jobs/wal/utils/SerializeUtil.scala | 2 +- .../s2jobs/wal/udfs/HFileParserUDFTest.scala | 78 ------------------- .../wal/utils/DeserializeUtilTest.scala | 4 +- 9 files changed, 73 insertions(+), 150 deletions(-) create mode 100644 s2core/src/main/scala/org/apache/s2graph/core/schema/SchemaManager.scala delete mode 100644 s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/ColumnSchema.scala delete mode 100644 s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/LabelSchema.scala rename s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/{SchemaManager.scala => MemorySchemaManager.scala} (93%) delete mode 100644 s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala diff --git a/s2core/src/main/scala/org/apache/s2graph/core/schema/SchemaManager.scala b/s2core/src/main/scala/org/apache/s2graph/core/schema/SchemaManager.scala new file mode 100644 index 00000000..19ebd6ec --- /dev/null +++ b/s2core/src/main/scala/org/apache/s2graph/core/schema/SchemaManager.scala @@ -0,0 +1,59 @@ +package org.apache.s2graph.core.schema + + +trait SchemaManager { + def findColumnService(columnId: Int): Service + + def findServiceColumn(columnId: Int): ServiceColumn + + def findServiceColumnMeta(columnId: Int, metaSeq: Byte): ColumnMeta + + def findLabel(labelId: Int): Label + + def findLabelService(labelId: Int): Service + + def findLabelIndex(labelId: Int, indexSeq: Byte): LabelIndex + + def findLabelIndexLabelMetas(labelId: Int, indexSeq: Byte): Array[LabelMeta] + + def findLabelMetas(labelId: Int): Map[Byte, LabelMeta] + + def findLabelMeta(labelId: Int, metaSeq: Byte): LabelMeta + + def checkLabelExist(labelId: Int): Boolean + + def checkServiceColumnExist(columnId: Int): Boolean + + def findServiceColumn(serviceName: String, columnName: String): ServiceColumn + + def findColumnMetas(serviceName: String, columnName: String): Map[String, ColumnMeta] + + def findColumnMetas(column: ServiceColumn): Map[String, ColumnMeta] + + def findLabel(labelName: String): Label + + def findLabelService(labelName: String): Service + + def findLabelIndices(label: Label): Map[String, LabelIndex] + + def findLabelIndices(labelName: String): Map[String, LabelIndex] + + def findLabelIndex(labelName: String, indexName: String): LabelIndex + + def findLabelIndexLabelMetas(labelName: String, indexName: String): Array[LabelMeta] + + def findLabelMetas(labelName: String): Map[String, LabelMeta] + + def findLabelMeta(labelName: String, name: String): LabelMeta + + def checkLabelExist(labelName: String): Boolean + + def findSrcServiceColumn(label: Label): ServiceColumn + + def findTgtServiceColumn(label: Label): ServiceColumn + + def findSrcServiceColumn(labelName: String): ServiceColumn + + def findTgtServiceColumn(labelName: String): ServiceColumn +} + diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/ColumnSchema.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/ColumnSchema.scala deleted file mode 100644 index 3263e7f7..00000000 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/ColumnSchema.scala +++ /dev/null @@ -1,48 +0,0 @@ -package org.apache.s2graph.s2jobs.wal - -import org.apache.s2graph.core.schema._ - -case class ColumnSchema(columnService: Map[Int, Service], - columns: Map[Int, ServiceColumn], - columnMetas: Map[Int, Map[Int, ColumnMeta]]) { - - val serviceMap: Map[String, Service] = columnService.map(t => t._2.serviceName -> t._2) - val serviceColumnMap: Map[(String, String), ServiceColumn] = { - columns.map { case (colId, column) => - val key = columnService(colId).serviceName -> column.columnName - - key -> column - } - } - val serviceColumnList: Map[String, Set[ServiceColumn]] = columnService.groupBy(_._2).map { case (service, ls) => - val columnIds = ls.map(_._1) - - service.serviceName -> columnIds.map(columns).toSet - } - val serviceColumnMetas: Map[(String, String), Map[String, ColumnMeta]] = columnMetas.map { case (columId, m) => - val service = columnService(columId) - val columnName = columns(columId).columnName - - val rev = m.map { case (seq, columnMeta) => - columnMeta.name -> columnMeta - } - - (service.serviceName, columnName) -> rev - } - - def findService(serviceName: String): Service = { - serviceMap(serviceName) - } - - def findServiceColumns(serviceName: String): Set[ServiceColumn] = { - serviceColumnList(serviceName) - } - - def findServiceColumn(serviceName: String, columnName: String): ServiceColumn = { - serviceColumnMap(serviceName -> columnName) - } - - def findServiceColumnMeta(serviceName: String, columnName: String, name: String): ColumnMeta = { - serviceColumnMetas(serviceName -> columnName)(name) - } -} diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/LabelSchema.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/LabelSchema.scala deleted file mode 100644 index d260b32f..00000000 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/LabelSchema.scala +++ /dev/null @@ -1,10 +0,0 @@ -package org.apache.s2graph.s2jobs.wal - -import org.apache.s2graph.core.schema._ - - -case class LabelSchema(labelService: Map[Int, String], - labels: Map[Int, Label], - labelIndices: Map[Int, Map[Byte, LabelIndex]], - labelIndexLabelMetas: Map[Int, Map[Byte, Array[LabelMeta]]], - labelMetas: Map[Int, Map[Byte, LabelMeta]]) \ No newline at end of file diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SchemaManager.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/MemorySchemaManager.scala similarity index 93% rename from s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SchemaManager.scala rename to s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/MemorySchemaManager.scala index 357d1e3e..b9d15961 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/SchemaManager.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/MemorySchemaManager.scala @@ -2,12 +2,12 @@ package org.apache.s2graph.s2jobs.wal import org.apache.s2graph.core.schema._ -class SchemaManager(serviceLs: Seq[Service], - serviceColumnLs: Seq[ServiceColumn], - columnMetaLs: Seq[ColumnMeta], - labelLs: Seq[Label], - labelIndexLs: Seq[LabelIndex], - labelMetaLs: Seq[LabelMeta]) { +class MemorySchemaManager(serviceLs: Seq[Service], + serviceColumnLs: Seq[ServiceColumn], + columnMetaLs: Seq[ColumnMeta], + labelLs: Seq[Label], + labelIndexLs: Seq[LabelIndex], + labelMetaLs: Seq[LabelMeta]) extends SchemaManager { private val services = serviceLs.map { service => service.serviceName -> service }.toMap private val servicesInverted = serviceLs.map { service => service.id.get -> service }.toMap @@ -244,13 +244,13 @@ class SchemaManager(serviceLs: Seq[Service], } } -object SchemaManager { +object MemorySchemaManager { def apply(serviceLs: Seq[Service], serviceColumnLs: Seq[ServiceColumn], columnMetaLs: Seq[ColumnMeta], labelLs: Seq[Label], labelIndexLs: Seq[LabelIndex], labelMetaLs: Seq[LabelMeta]): SchemaManager = { - new SchemaManager(serviceLs, serviceColumnLs, columnMetaLs, labelLs, labelIndexLs, labelMetaLs) + new MemorySchemaManager(serviceLs, serviceColumnLs, columnMetaLs, labelLs, labelIndexLs, labelMetaLs) } } diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala index 74431883..0e03080a 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtil.scala @@ -3,7 +3,7 @@ package org.apache.s2graph.s2jobs.wal.utils import org.apache.hadoop.hbase.client.Result import org.apache.hadoop.hbase.util.Bytes import org.apache.hadoop.hbase.{Cell, CellUtil} -import org.apache.s2graph.core.schema.{ColumnMeta, LabelMeta, ServiceColumn} +import org.apache.s2graph.core.schema._ import org.apache.s2graph.core.storage.SKeyValue import org.apache.s2graph.core.storage.serde.StorageDeserializable import org.apache.s2graph.core.storage.serde.StorageDeserializable.bytesToInt diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala index f1f4bbf1..c47874da 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SchemaUtil.scala @@ -3,7 +3,7 @@ package org.apache.s2graph.s2jobs.wal.utils import com.typesafe.config.Config import org.apache.s2graph.core.schema._ import org.apache.s2graph.core.utils.logger -import org.apache.s2graph.s2jobs.wal.SchemaManager +import org.apache.s2graph.s2jobs.wal.MemorySchemaManager object SchemaUtil { @@ -102,7 +102,7 @@ object SchemaUtil { label.metaProps } - SchemaManager(serviceLs, serviceColumnLs, columnMetaLs, labelLs, labelIndexLs, labelMetaLs) + MemorySchemaManager(serviceLs, serviceColumnLs, columnMetaLs, labelLs, labelIndexLs, labelMetaLs) } def buildVertexDeserializeSchema(serviceNameColumnNames: Map[String, Seq[String]]): SchemaManager = { diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala index 2f54512e..bf5b217d 100644 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/wal/utils/SerializeUtil.scala @@ -8,7 +8,7 @@ import org.apache.s2graph.core.schema._ import org.apache.s2graph.core.storage.SKeyValue import org.apache.s2graph.core.storage.serde.Serializable import org.apache.s2graph.core.types._ -import org.apache.s2graph.s2jobs.wal.{SchemaManager, WalLog, WalVertex} +import org.apache.s2graph.s2jobs.wal.{MemorySchemaManager, WalLog, WalVertex} import play.api.libs.json.{JsObject, Json} object SerializeUtil { diff --git a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala deleted file mode 100644 index f53379b8..00000000 --- a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/udfs/HFileParserUDFTest.scala +++ /dev/null @@ -1,78 +0,0 @@ -/* -package org.apache.s2graph.s2jobs.wal.udfs - -import com.holdenkarau.spark.testing.DataFrameSuiteBase -import com.typesafe.config.ConfigFactory -import org.apache.hadoop.hbase.client.Result -import org.apache.s2graph.core.storage.SKeyValue -import org.apache.s2graph.s2jobs.task._ -import org.apache.s2graph.s2jobs.wal.utils.{DeserializeUtil, SchemaUtil} -import org.apache.spark.sql.{Encoders, Row, SaveMode} -import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers} - -import scala.reflect.runtime.universe._ - -class HFileParserUDFTest extends FunSuite with Matchers with BeforeAndAfterAll with DataFrameSuiteBase { - - test("transform") { - import spark.implicits._ -// implicit val tt = typeTag[HResult] -// implicit val enc = Encoders.kryo[HResult] - - val outputPath = "transform.parquet" - val df = - spark.read.parquet("direct.parquet").limit(100) - - val results = df.map { row => - val kvs = row.getAs[Seq[Row]]("kvs").map { row => - DeserializeUtil.sKeyValueFromRow(row) - } - - val table = kvs.head.table - val rowKey = kvs.head.row - val cf = kvs.head.cf - - val partialKVs = kvs.map { kv => PartialKeyValue(kv.qualifier, kv.value, kv.timestamp) } - HResult(table, rowKey, cf, partialKVs) - } - - results.write.mode(SaveMode.Overwrite).parquet(outputPath) - } - - test("deserialize result udf.") { - import spark.implicits._ - - val sKeyValueParser = new HFileParserUDF - sKeyValueParser.register(spark, "to_wal", Map( - "elementType" -> "indexedge", - "labelNames" -> "talk_friend", - "db.default.url" -> "jdbc:mysql://comm-s2graph.mydb.iwilab.com/graph", - "db.default.driver" -> "com.mysql.jdbc.Driver", - "db.default.user" -> "graph", - "db.default.password" -> "graph" - )) - - val df = - spark.read.parquet("transform.parquet") - - df.printSchema() - df.createOrReplaceTempView("results") - - - val sql = - """ - |SELECT wal.* - |FROM ( - | SELECT explode(to_wal(row, kvs)) as wal - | FROM results - |) - """.stripMargin - - - val result = spark.sql(sql) - result.printSchema() - - result.write.mode(SaveMode.Overwrite).json("./parsed.json") - } -} -*/ diff --git a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala index 115f4893..3d6e45ab 100644 --- a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala +++ b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/wal/utils/DeserializeUtilTest.scala @@ -5,7 +5,7 @@ import org.apache.hadoop.hbase.KeyValue.Type import org.apache.hadoop.hbase.client.Result import org.apache.s2graph.core.schema._ import org.apache.s2graph.core.storage.SKeyValue -import org.apache.s2graph.s2jobs.wal.{SchemaManager, WalLog, WalVertex} +import org.apache.s2graph.s2jobs.wal.{MemorySchemaManager, WalLog, WalVertex} import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers} import play.api.libs.json.{JsObject, JsValue, Json} @@ -32,7 +32,7 @@ class DeserializeUtilTest extends FunSuite with Matchers with BeforeAndAfterAll val labelIndex = LabelIndex(Option(1), label.id.get, "pk", 1, Seq(labelMeta.seq), "", None, None) val labelIndexLs = Seq(labelIndex) - SchemaManager(serviceLs, + MemorySchemaManager(serviceLs, serviceColumnLs, columnMetaLs, labelLs,