Skip to content

Commit

Permalink
Merge pull request #521 from Kotlin/linter-for-all-modules
Browse files Browse the repository at this point in the history 
Linter for all modules
  • Loading branch information
@Jolanrensen
Jolanrensen authored Nov 30, 2023
2 parents b3f3331 + d50002b commit f36912c
Show file tree
Hide file tree
Showing 36 changed files with 919 additions and 529 deletions.
27 changes: 27 additions & 0 deletions build.gradle.kts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
import org.jetbrains.kotlin.tooling.core.closure
import org.jetbrains.kotlinx.publisher.apache2
import org.jetbrains.kotlinx.publisher.developer
import org.jetbrains.kotlinx.publisher.githubRepo
import org.jmailen.gradle.kotlinter.KotlinterExtension

@Suppress("DSL_SCOPE_VIOLATION", "UnstableApiUsage")
plugins {
Expand Down Expand Up @@ -51,6 +53,31 @@ allprojects {
sourceCompatibility = JavaVersion.VERSION_1_8.toString()
targetCompatibility = JavaVersion.VERSION_1_8.toString()
}

// Attempts to configure kotlinter for each sub-project that uses the plugin
afterEvaluate {
try {
kotlinter {
ignoreFailures = false
reporters = arrayOf("checkstyle", "plain")
experimentalRules = true
disabledRules = arrayOf(
"no-wildcard-imports",
"experimental:spacing-between-declarations-with-annotations",
"experimental:enum-entry-name-case",
"experimental:argument-list-wrapping",
"experimental:annotation",
"max-line-length",
"filename",
"comment-spacing",
"curly-spacing",
"experimental:annotation-spacing"
)
}
} catch (_: UnknownDomainObjectException) {
logger.warn("Could not set kotlinter config on :${this.name}")
}
}
}

koverMerged {
Expand Down
18 changes: 0 additions & 18 deletions core/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,24 +298,6 @@ korro {
}
}

kotlinter {
ignoreFailures = false
reporters = arrayOf("checkstyle", "plain")
experimentalRules = true
disabledRules = arrayOf(
"no-wildcard-imports",
"experimental:spacing-between-declarations-with-annotations",
"experimental:enum-entry-name-case",
"experimental:argument-list-wrapping",
"experimental:annotation",
"max-line-length",
"filename",
"comment-spacing",
"curly-spacing",
"experimental:annotation-spacing"
)
}

tasks.withType<KspTaskJvm> {
dependsOn(tasks.generateKeywordsSrc)
}
Expand Down
4 changes: 2 additions & 2 deletions ...ed-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterHtmlRenderer.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,9 @@ internal inline fun <reified T : Any> JupyterHtmlRenderer.render(
includeCss = true,
).plus(
df.toHTML(
configuration = reifiedDisplayConfiguration,
// is added later to make sure it's put outside of potential iFrames
configuration = reifiedDisplayConfiguration.copy(enableFallbackStaticTables = false),
cellRenderer = contextRenderer,
includeStatic = false, // is added later to make sure it's put outside of potential iFrames
) { footer }
).toJupyterHtmlData()

Expand Down
1 change: 1 addition & 0 deletions dataframe-arrow/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ plugins {
kotlin("jvm")
kotlin("libs.publisher")
id("org.jetbrains.kotlinx.kover")
id("org.jmailen.kotlinter")
}

group = "org.jetbrains.kotlinx"
Expand Down
168 changes: 139 additions & 29 deletions dataframe-arrow/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/ArrowWriterImpl.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ import org.jetbrains.kotlinx.dataframe.exceptions.CellConversionException
import org.jetbrains.kotlinx.dataframe.exceptions.TypeConverterNotFoundException
import org.jetbrains.kotlinx.dataframe.name
import org.jetbrains.kotlinx.dataframe.values
import java.nio.charset.Charset
import kotlin.reflect.full.isSubtypeOf
import kotlin.reflect.typeOf

Expand Down Expand Up @@ -85,15 +84,21 @@ internal class ArrowWriterImpl(
private fun countTotalBytes(column: AnyCol): Long? {
val columnType = column.type()
return when {
columnType.isSubtypeOf(typeOf<String?>()) -> column.values.fold(0L) {totalBytes, value -> totalBytes + value.toString().length * 4}
columnType.isSubtypeOf(typeOf<String?>()) -> column.values.fold(0L) { totalBytes, value -> totalBytes + value.toString().length * 4 }
else -> null
}
}

private fun infillWithNulls(vector: FieldVector, size: Int) {
when (vector) {
is BaseFixedWidthVector -> for (i in 0 until size) { vector.setNull(i) }
is BaseVariableWidthVector -> for (i in 0 until size) { vector.setNull(i) }
is BaseFixedWidthVector -> for (i in 0 until size) {
vector.setNull(i)
}

is BaseVariableWidthVector -> for (i in 0 until size) {
vector.setNull(i)
}

else -> throw IllegalArgumentException("Can not infill ${vector.javaClass.canonicalName}")
}
vector.valueCount = size
Expand All @@ -110,7 +115,8 @@ internal class ArrowWriterImpl(
ArrowType.Int(32, true) -> column.convertToInt()
ArrowType.Int(64, true) -> column.convertToLong()
is ArrowType.Decimal -> column.convertToBigDecimal()
ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) -> column.convertToDouble().convertToFloat() // Use [convertToDouble] as locale logic step
ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) -> column.convertToDouble()
.convertToFloat() // Use [convertToDouble] as locale logic step
ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE) -> column.convertToDouble()
ArrowType.Date(DateUnit.DAY) -> column.convertToLocalDate()
ArrowType.Date(DateUnit.MILLISECOND) -> column.convertToLocalDateTime()
Expand All @@ -123,25 +129,108 @@ internal class ArrowWriterImpl(

private fun infillVector(vector: FieldVector, column: AnyCol) {
when (vector) {
is VarCharVector -> column.convertToString().forEachIndexed { i, value -> value?.let { vector.set(i, Text(value)); value } ?: vector.setNull(i) }
is LargeVarCharVector -> column.convertToString().forEachIndexed { i, value -> value?.let { vector.set(i, Text(value)); value } ?: vector.setNull(i) }
is BitVector -> column.convertToBoolean().forEachIndexed { i, value -> value?.let { vector.set(i, value.compareTo(false)); value } ?: vector.setNull(i) }
is TinyIntVector -> column.convertToInt().forEachIndexed { i, value -> value?.let { vector.set(i, value); value } ?: vector.setNull(i) }
is SmallIntVector -> column.convertToInt().forEachIndexed { i, value -> value?.let { vector.set(i, value); value } ?: vector.setNull(i) }
is IntVector -> column.convertToInt().forEachIndexed { i, value -> value?.let { vector.set(i, value); value } ?: vector.setNull(i) }
is BigIntVector -> column.convertToLong().forEachIndexed { i, value -> value?.let { vector.set(i, value); value } ?: vector.setNull(i) }
is DecimalVector -> column.convertToBigDecimal().forEachIndexed { i, value -> value?.let { vector.set(i, value); value } ?: vector.setNull(i) }
is Decimal256Vector -> column.convertToBigDecimal().forEachIndexed { i, value -> value?.let { vector.set(i, value); value } ?: vector.setNull(i) }
is Float8Vector -> column.convertToDouble().forEachIndexed { i, value -> value?.let { vector.set(i, value); value } ?: vector.setNull(i) }
is Float4Vector -> column.convertToFloat().forEachIndexed { i, value -> value?.let { vector.set(i, value); value } ?: vector.setNull(i) }

is DateDayVector -> column.convertToLocalDate().forEachIndexed { i, value -> value?.let { vector.set(i, (value.toJavaLocalDate().toEpochDay()).toInt()); value } ?: vector.setNull(i) }
is DateMilliVector -> column.convertToLocalDateTime().forEachIndexed { i, value -> value?.let { vector.set(i, value.toInstant(
TimeZone.UTC).toEpochMilliseconds()); value } ?: vector.setNull(i) }
is TimeNanoVector -> column.convertToLocalTime().forEachIndexed { i, value -> value?.let { vector.set(i, value.toNanoOfDay()); value } ?: vector.setNull(i) }
is TimeMicroVector -> column.convertToLocalTime().forEachIndexed { i, value -> value?.let { vector.set(i, value.toNanoOfDay() / 1000); value } ?: vector.setNull(i) }
is TimeMilliVector -> column.convertToLocalTime().forEachIndexed { i, value -> value?.let { vector.set(i, (value.toNanoOfDay() / 1000 / 1000).toInt()); value } ?: vector.setNull(i) }
is TimeSecVector -> column.convertToLocalTime().forEachIndexed { i, value -> value?.let { vector.set(i, (value.toNanoOfDay() / 1000 / 1000 / 1000).toInt()); value } ?: vector.setNull(i) }
is VarCharVector -> column.convertToString()
.forEachIndexed { i, value ->
value?.also { vector.set(i, Text(value)) }
?: vector.setNull(i)
}

is LargeVarCharVector -> column.convertToString()
.forEachIndexed { i, value ->
value?.also { vector.set(i, Text(value)) }
?: vector.setNull(i)
}

is BitVector -> column.convertToBoolean()
.forEachIndexed { i, value ->
value?.also { vector.set(i, value.compareTo(false)) }
?: vector.setNull(i)
}

is TinyIntVector -> column.convertToInt()
.forEachIndexed { i, value ->
value?.also { vector.set(i, value) }
?: vector.setNull(i)
}

is SmallIntVector -> column.convertToInt()
.forEachIndexed { i, value ->
value?.also { vector.set(i, value) }
?: vector.setNull(i)
}

is IntVector -> column.convertToInt()
.forEachIndexed { i, value ->
value?.also { vector.set(i, value) }
?: vector.setNull(i)
}

is BigIntVector -> column.convertToLong()
.forEachIndexed { i, value ->
value?.also { vector.set(i, value) }
?: vector.setNull(i)
}

is DecimalVector -> column.convertToBigDecimal()
.forEachIndexed { i, value ->
value?.also { vector.set(i, value) }
?: vector.setNull(i)
}

is Decimal256Vector -> column.convertToBigDecimal()
.forEachIndexed { i, value ->
value?.also { vector.set(i, value) }
?: vector.setNull(i)
}

is Float8Vector -> column.convertToDouble()
.forEachIndexed { i, value ->
value?.also { vector.set(i, value) }
?: vector.setNull(i)
}

is Float4Vector -> column.convertToFloat()
.forEachIndexed { i, value ->
value?.also { vector.set(i, value) }
?: vector.setNull(i)
}

is DateDayVector -> column.convertToLocalDate()
.forEachIndexed { i, value ->
value?.also { vector.set(i, value.toJavaLocalDate().toEpochDay().toInt()) }
?: vector.setNull(i)
}

is DateMilliVector -> column.convertToLocalDateTime()
.forEachIndexed { i, value ->
value?.also { vector.set(i, value.toInstant(TimeZone.UTC).toEpochMilliseconds()) }
?: vector.setNull(i)
}

is TimeNanoVector -> column.convertToLocalTime()
.forEachIndexed { i, value ->
value?.also { vector.set(i, value.toNanoOfDay()) }
?: vector.setNull(i)
}

is TimeMicroVector -> column.convertToLocalTime()
.forEachIndexed { i, value ->
value?.also { vector.set(i, value.toNanoOfDay() / 1000) }
?: vector.setNull(i)
}

is TimeMilliVector -> column.convertToLocalTime()
.forEachIndexed { i, value ->
value?.also { vector.set(i, (value.toNanoOfDay() / 1000 / 1000).toInt()) }
?: vector.setNull(i)
}

is TimeSecVector -> column.convertToLocalTime()
.forEachIndexed { i, value ->
value?.also { vector.set(i, (value.toNanoOfDay() / 1000 / 1000 / 1000).toInt()) }
?: vector.setNull(i)
}

else -> {
// TODO implement other vector types from [readField] (VarBinaryVector, UIntVector, DurationVector, StructVector) and may be others (ListVector, FixedSizeListVector etc)
throw NotImplementedError("Saving to ${vector.javaClass.canonicalName} is currently not implemented")
Expand All @@ -154,7 +243,12 @@ internal class ArrowWriterImpl(
/**
* Create Arrow FieldVector with [column] content cast to [field] type according to [strictType] and [strictNullable] settings.
*/
private fun allocateVectorAndInfill(field: Field, column: AnyCol?, strictType: Boolean, strictNullable: Boolean): FieldVector {
private fun allocateVectorAndInfill(
field: Field,
column: AnyCol?,
strictType: Boolean,
strictNullable: Boolean,
): FieldVector {
val containNulls = (column == null || column.hasNulls())
// Convert the column to type specified in field. (If we already have target type, convertTo will do nothing)

Expand All @@ -163,12 +257,19 @@ internal class ArrowWriterImpl(
} catch (e: CellConversionException) {
if (strictType) {
// If conversion failed but strictType is enabled, throw the exception
val mismatch = ConvertingMismatch.TypeConversionFail.ConversionFailError(e.column?.name() ?: "", e.row, e)
val mismatch =
ConvertingMismatch.TypeConversionFail.ConversionFailError(e.column?.name() ?: "", e.row, e)
mismatchSubscriber(mismatch)
throw ConvertingException(mismatch)
} else {
// If strictType is not enabled, use original data with its type. Target nullable is saved at this step.
mismatchSubscriber(ConvertingMismatch.TypeConversionFail.ConversionFailIgnored(e.column?.name() ?: "", e.row, e))
mismatchSubscriber(
ConvertingMismatch.TypeConversionFail.ConversionFailIgnored(
e.column?.name() ?: "",
e.row,
e
)
)
column to column!!.toArrowField(mismatchSubscriber)
}
} catch (e: TypeConverterNotFoundException) {
Expand Down Expand Up @@ -197,8 +298,17 @@ internal class ArrowWriterImpl(
mismatchSubscriber(mismatch)
throw ConvertingException(mismatch)
} else {
mismatchSubscriber(ConvertingMismatch.NullableMismatch.NullValueIgnored(actualField.name, firstNullValue))
Field(actualField.name, FieldType(true, actualField.fieldType.type, actualField.fieldType.dictionary), actualField.children).createVector(allocator)!!
mismatchSubscriber(
ConvertingMismatch.NullableMismatch.NullValueIgnored(
actualField.name,
firstNullValue
)
)
Field(
actualField.name,
FieldType(true, actualField.fieldType.type, actualField.fieldType.dictionary),
actualField.children
).createVector(allocator)!!
}
} else {
actualField.createVector(allocator)!!
Expand Down
Loading

0 comments on commit f36912c

Please sign in to comment.