feature incremental api generator, small fix string generator #26

Open
sergey.vlasenko wants to merge 4 commits from incremental_api_generator into feature/MB-43110_incremental_api_generator
11 changed files with 441 additions and 42 deletions

View File

@ -6,6 +6,7 @@ import org.gradle.api.Project
import org.gradle.api.tasks.compile.JavaCompile import org.gradle.api.tasks.compile.JavaCompile
import org.gradle.kotlin.dsl.findByType import org.gradle.kotlin.dsl.findByType
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
import static_analysis.utils.getApiGeneratorExtension
class ApiGeneratorAndroidPlugin : ApiGeneratorPlugin() { class ApiGeneratorAndroidPlugin : ApiGeneratorPlugin() {
@ -13,11 +14,10 @@ class ApiGeneratorAndroidPlugin : ApiGeneratorPlugin() {
super.apply(target) super.apply(target)
with(target) { with(target) {
val extension = getExtension() val extension = getApiGeneratorExtension()
val outputDir = getDirectoryForGeneration() val outputDir = getDirectoryForGeneration()
extension.outputDirPath = outputDir.path extension.outputDirPath = outputDir.path
extension.recreateOutputDir = true
afterEvaluate { afterEvaluate {
extensions.findByType<LibraryExtension>()?.apply { extensions.findByType<LibraryExtension>()?.apply {

View File

@ -1,13 +1,14 @@
package apigen package apigen
import org.gradle.api.Project import org.gradle.api.Project
import static_analysis.utils.getApiGeneratorExtension
class ApiGeneratorBackendPlugin : ApiGeneratorPlugin() { class ApiGeneratorBackendPlugin : ApiGeneratorPlugin() {
override fun apply(target: Project) { override fun apply(target: Project) {
super.apply(target) super.apply(target)
val extension = target.getExtension() val extension = target.getApiGeneratorExtension()
extension.outputDirPath = target.file("src/main/kotlin").path extension.outputDirPath = target.file("src/main/kotlin").path
extension.recreateOutputDir = false extension.recreateOutputDir = false

View File

@ -2,8 +2,10 @@ package apigen
import org.gradle.api.Plugin import org.gradle.api.Plugin
import org.gradle.api.Project import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.kotlin.dsl.create import org.gradle.kotlin.dsl.create
import org.gradle.kotlin.dsl.dependencies import org.gradle.kotlin.dsl.dependencies
import org.gradle.kotlin.dsl.register
import org.gradle.kotlin.dsl.repositories import org.gradle.kotlin.dsl.repositories
abstract class ApiGeneratorPlugin : Plugin<Project> { abstract class ApiGeneratorPlugin : Plugin<Project> {
@ -12,6 +14,8 @@ abstract class ApiGeneratorPlugin : Plugin<Project> {
const val API_GENERATOR_CONFIG = "apiGenerator" const val API_GENERATOR_CONFIG = "apiGenerator"
const val API_GENERATOR_EXT_NAME = "apiGenerator" const val API_GENERATOR_EXT_NAME = "apiGenerator"
const val API_GENERATOR_DEFAULT_VERSION = "1.4.0-beta10" const val API_GENERATOR_DEFAULT_VERSION = "1.4.0-beta10"
const val API_GENERATOR_CLEAN_TEMP_DIR_TASK = "apiGeneratorCleanTempDir"
} }
override fun apply(target: Project) { override fun apply(target: Project) {
@ -33,7 +37,8 @@ abstract class ApiGeneratorPlugin : Plugin<Project> {
extensions.create<ApiGeneratorExtension>(API_GENERATOR_EXT_NAME) extensions.create<ApiGeneratorExtension>(API_GENERATOR_EXT_NAME)
val apiGenTask = createApiGeneratorTask() val cleanTask = tasks.create(API_GENERATOR_CLEAN_TEMP_DIR_TASK)
val apiGenTask = getApiGenTask(cleanTask)
gradle.projectsEvaluated { gradle.projectsEvaluated {
tasks.getByName("preBuild").dependsOn(apiGenTask) tasks.getByName("preBuild").dependsOn(apiGenTask)
@ -41,42 +46,15 @@ abstract class ApiGeneratorPlugin : Plugin<Project> {
} }
} }
protected fun Project.getExtension(): ApiGeneratorExtension = extensions.getByName(API_GENERATOR_EXT_NAME) as ApiGeneratorExtension private fun Project.getApiGenTask(cleanTask: Task) = tasks.register(API_GENERATOR_CONFIG, ApiGeneratorTask::class) {
val apiGeneratorTempDir = temporaryDir
tempApiDirDirectory.set(apiGeneratorTempDir.absolutePath)
private fun Project.createApiGeneratorTask() = tasks.register(API_GENERATOR_CONFIG) { // Нужно для удаления временной директории даже в случае краша во время выполнения таски.
val extension = getExtension() finalizedBy(
cleanTask.doFirst {
val pathToApiSchemes = extension.pathToApiSchemes ?: throw IllegalStateException("Configure path to api schemes for api generator plugin") apiGeneratorTempDir.deleteRecursively()
val outputLanguage = extension.outputLanguage ?: throw IllegalStateException("Configure output language code for api generator plugin") }
)
outputs.files(files(pathToApiSchemes).asFileTree.files)
doLast {
javaexec {
mainClass.set("-jar")
workingDir = rootDir
args = listOfNotNull(
"--add-opens",
"java.base/java.lang=ALL-UNNAMED",
"--add-opens",
"java.base/java.time=ALL-UNNAMED",
configurations.getByName("apiGenerator").asPath,
"generate-client-code",
"--output-language",
outputLanguage.argName,
"--specification-path",
pathToApiSchemes,
"--kotlin-methods-generation-mode".takeIf { outputLanguage.methodOutputType != null },
outputLanguage.methodOutputType?.argName,
"--output-path",
extension.outputDirPath,
"--package-name",
extension.outputPackageName,
"--recreate_output_dirs",
extension.recreateOutputDir.toString(),
)
}
}
} }
} }

View File

@ -0,0 +1,157 @@
package apigen
import apigen.depencency_resolver.ApiModelsDependencyResolver
import apigen.depencency_resolver.CopyApiModelsToTempDirTask
import apigen.depencency_resolver.FileParser
import org.gradle.api.DefaultTask
import org.gradle.api.file.FileTree
import org.gradle.api.file.FileType
import org.gradle.api.provider.Property
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFiles
import org.gradle.api.tasks.Internal
import org.gradle.api.tasks.OutputFiles
import org.gradle.api.tasks.TaskAction
import org.gradle.work.ChangeType
import org.gradle.work.Incremental
import org.gradle.work.InputChanges
import org.gradle.workers.WorkerExecutor
import static_analysis.utils.getApiGeneratorExtension
import static_analysis.utils.toPath
import java.io.File
import java.nio.file.Path
import javax.inject.Inject
abstract class ApiGeneratorTask: DefaultTask() {
@Internal
val extension: ApiGeneratorExtension = project.getApiGeneratorExtension()
@Internal
val pathToApiScheme: String = extension.pathToApiSchemes
?: throw IllegalStateException("Configure path to api schemes for api generator plugin")
@Internal
val outputLanguage: OutputLanguage = extension.outputLanguage
?: throw IllegalStateException("Configure output language code for api generator plugin")
@Inject
abstract fun getWorkerExecutor(): WorkerExecutor
@Incremental
@InputFiles
val inputFiles: FileTree = project.files(pathToApiScheme).asFileTree
@OutputFiles
val outputFiles: FileTree = project.files(extension.outputDirPath).asFileTree
@get:Input
abstract val tempApiDirDirectory: Property<String>
@TaskAction
fun execute(inputChanges: InputChanges): Unit = with(project) {
if (inputChanges.isIncremental) {
handleInputChanges(inputChanges)
} else {
generateApi(
pathToApi = pathToApiScheme,
outputLanguage = outputLanguage,
recreateOutputDir = true
)
}
}
private fun generateApi(
pathToApi: String,
outputLanguage: OutputLanguage,
recreateOutputDir: Boolean
) = with(project) {
javaexec {
main = "-jar"
workingDir = rootDir
args = listOfNotNull(
configurations.getByName("apiGenerator").asPath,
"generate-client-code",
"--output-language",
outputLanguage.argName,
"--specification-path",
pathToApi,
"--kotlin-methods-generation-mode".takeIf { outputLanguage.methodOutputType != null },
outputLanguage.methodOutputType?.argName,
"--output-path",
extension.outputDirPath,
"--package-name",
extension.outputPackageName,
"--recreate_output_dirs",
recreateOutputDir.toString()
)
}
}
private fun handleInputChanges(inputChanges: InputChanges) {
val modifiedFiles = hashSetOf<File>()
inputChanges.getFileChanges(inputFiles)
.filter { it.fileType == FileType.FILE }
.forEach { change ->
Review

change ->

file ->

change -> file ->
when (change.changeType) {
ChangeType.REMOVED -> {
outputFiles.find { it.nameWithoutExtension == change.file.nameWithoutExtension }?.delete()
}
else -> modifiedFiles.add(change.file)
}
}
if (modifiedFiles.isEmpty()) return
handleModifiedFiles(modifiedFiles)
}
private fun handleModifiedFiles(modifiedFiles: MutableCollection<File>) {
val dependencyResolver = ApiModelsDependencyResolver(
pathToApi = pathToApiScheme,
fileParser = FileParser()
)
modifiedFiles.addAll(
dependencyResolver.getDependenciesFromFiles(modifiedFiles)
)
val tempApiDirectory = tempApiDirDirectory.getOrElse("")
if (tempApiDirectory.isBlank()) {
throw IllegalStateException("TempApiDirDirectory is blank or not specified.")
}
copyAllFilesToTempDir(
tempDirPath = tempApiDirectory,
files = modifiedFiles
)
generateApi(
pathToApi = tempApiDirectory,
outputLanguage = outputLanguage,
recreateOutputDir = false
)
}
private fun copyAllFilesToTempDir(tempDirPath: String, files: Collection<File>) {
val workQueue = getWorkerExecutor().noIsolation()
files.forEach { file ->
val newFilePath = pathToApiScheme.toPath()
.relativize(file.absolutePath.toPath()).toString()
val newFileFullPath = Path.of(tempDirPath, newFilePath).toString()
workQueue.submit(CopyApiModelsToTempDirTask::class.java) {
getSourceFile().set(file)
getTargetFilePath().set(newFileFullPath)
}
}
workQueue.await()
}
}

View File

@ -0,0 +1,71 @@
package apigen.depencency_resolver
import apigen.depencency_resolver.FileParser.Companion.NAME_JSON_KEY
import groovy.json.JsonSlurper
import java.io.File
/**
* Класс для разрешения зависимостей АПИ моделей от других АПИ моделей.
*
* @param pathToApi путь к директории АПИ, где будут искаться зависимости
* @param fileParser класс, ответственный за поиск типов в АПИ модельке.
*/
@Suppress("UNCHECKED_CAST")
class ApiModelsDependencyResolver(
pathToApi: String,
private val fileParser: FileParser
) {
// Мапа для хранения и быстрого поиска типа модельки по файлам, т.к. не всегда имя файла совпадает с типом модельки.
Review

c -- у тебя на латинице

c -- у тебя на латинице
private val modelTypes: HashMap<String, File> = hashMapOf()
private val handledFiles = hashSetOf<File>()
init {
File(pathToApi)
.walk()
.filter { it.isFile }
.forEach { file ->
val json = JsonSlurper().parse(file) as? Map<String, Any?>
val type = json?.getOrDefault(NAME_JSON_KEY, file.nameWithoutExtension).toString()
modelTypes[type] = file
}
}
fun getDependenciesFromFiles(files: Collection<File>): Set<File> =
files.fold(setOf()) { foundDependencies, file ->
val newDependencies = getFileDependencies(file)
if (newDependencies.isEmpty()) {
foundDependencies
} else {
foundDependencies + getFileDependencies(file)
}
}
private fun getFileDependencies(sourceFile: File): Set<File> {
if (handledFiles.contains(sourceFile)) {
return emptySet()
}
handledFiles.add(sourceFile)
val json = JsonSlurper().parse(sourceFile) as Map<String, Any?>
val files = fileParser.getAllTypeNamesFromJson(json)
.map { type ->
modelTypes[type]
?: throw IllegalArgumentException("Couldn't resolve $type in ${sourceFile.name}")
}
.toSet()
return files.fold(files) { foundDependencies, file ->
val newDependencies = getFileDependencies(file)
if (newDependencies.isEmpty()) {
foundDependencies
} else {
foundDependencies + newDependencies
}
Review

files.fold(files) { foundDependencies, file ->
val newDependencies = getFileDependencies(file)

        if (newDependencies.isEmpty()) {
            foundDependencies
        } else {
            foundDependencies + newDependencies
        }

такой же код есть в getDependenciesFromFiles

files.fold(files) { foundDependencies, file -> val newDependencies = getFileDependencies(file) if (newDependencies.isEmpty()) { foundDependencies } else { foundDependencies + newDependencies } такой же код есть в getDependenciesFromFiles
}
}
}

View File

@ -0,0 +1,22 @@
package apigen.depencency_resolver
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.provider.Property
import org.gradle.workers.WorkAction
import org.gradle.workers.WorkParameters
import java.io.File
abstract class CopyApiModelsToTempDirTask: WorkAction<CopyApiModelsToTempDirTaskParams> {
override fun execute() {
val sourceFile = parameters.getSourceFile().asFile.get()
val target = File(parameters.getTargetFilePath().get())
sourceFile.copyTo(target = target)
}
}
interface CopyApiModelsToTempDirTaskParams: WorkParameters {
fun getSourceFile(): RegularFileProperty
fun getTargetFilePath(): Property<String>
}

View File

@ -0,0 +1,103 @@
package apigen.depencency_resolver
@Suppress("UNCHECKED_CAST")
class FileParser {
companion object {
const val NAME_JSON_KEY = "name"
const val PARENT_JSON_KEY = "parent"
const val TYPE_JSON_KEY = "type"
}
private val defaultTypes = hashSetOf(
"Bool",
"String",
"DateTime",
"Long",
"Int",
"Double",
"Decimal",
"Date",
"DateTimeTimestamp",
"Color",
"StringDecimal",
"Url",
"Map",
"null" // Костыль, чтобы не писать проверки на null при получении типа из мапы
)
fun getAllTypeNamesFromJson(model: Map<String, Any?>): Set<String> {
val currentModelType = Type.getType(model[NAME_JSON_KEY].toString())
var generic = ""
if (currentModelType is TypeWithGeneric) {
generic = currentModelType.getGenericTypeName()
}
val parentRawTypeName = model[PARENT_JSON_KEY].toString()
return model.entries
.fold(setOf(parentRawTypeName)) { foundRawTypeNames: Set<String>, jsonEntry: Map.Entry<String, Any?> ->
foldDependencies(foundRawTypeNames, jsonEntry)
}
.minus(defaultTypes)
.minus(generic)
.fold(setOf()) { foundTypeNames, rawTypeName -> foundTypeNames + getTypeFromRawTypeNames(rawTypeName) }
}
private fun foldDependencies(foundTypes: Set<String>, jsonEntry: Map.Entry<String, Any?>): Set<String> {
val value = jsonEntry.value
val newTypes: Set<String> = when {
(value as? Collection<Map<String, Any?>>) != null -> {
Review

Из-за дженерика обычный приведение типов через is не работает

Из-за дженерика обычный приведение типов через `is` не работает
getTypeFromCollection(value)
}
(value as? Map<String, Any?>) != null && jsonEntry.key == TYPE_JSON_KEY -> {
val typeName = value[NAME_JSON_KEY].toString()
val parentTypeName = value[PARENT_JSON_KEY].toString()
setOf(typeName, parentTypeName)
}
else -> setOf()
}
return if (newTypes.isEmpty()) {
foundTypes
} else {
foundTypes + newTypes
}
Review

а зачем вообще конструкция
if (newTypes.isEmpty()) {
foundTypes
} else {
foundTypes + newTypes
}

если newTypes пустой, то foundTypes + newTypes как раз будет равен foundTypes
Разве нет?

а зачем вообще конструкция if (newTypes.isEmpty()) { foundTypes } else { foundTypes + newTypes } если newTypes пустой, то foundTypes + newTypes как раз будет равен foundTypes Разве нет?
}
private fun getTypeFromCollection(value: Collection<Map<String, Any?>>): Set<String> =
value.fold(setOf()) { types: Set<String>, collectionEntry: Map<String, Any?> ->
types + getAllTypeNamesFromJson(collectionEntry)
}
private fun getTypeFromRawTypeNames(typeName: String): Set<String> {
val type = Type.getType(typeName)
val newTypeNames = mutableSetOf<String>()
when (type) {
is SimpleType -> {
newTypeNames.add(type.getMainTypeName())
}
is ArrayType -> {
newTypeNames.add(type.getMainTypeName())
}
is MapType -> {
newTypeNames.addAll(type.getSubTypeNames())
Review

во всех остальных случая дубль кода

во всех остальных случая дубль кода
}
is TypeWithGeneric -> {
newTypeNames.add(type.getMainTypeName())
}
}
return newTypeNames
}
}

View File

@ -0,0 +1,57 @@
package apigen.depencency_resolver
sealed class Type(protected val typeName: String) {
companion object {
private val simpleTypeRegex = Regex("\\w*")
private val mapRegex = Regex("Map<.*?,.*?>")
private val arrayRegex = Regex(".*?\\[?]")
private val typeWithGenericRegex = Regex(".*?<.*?>")
fun getType(typeName: String): Type = when {
typeName.matches(mapRegex) -> MapType(typeName)
typeName.matches(arrayRegex) -> ArrayType(typeName)
typeName.matches(typeWithGenericRegex) -> TypeWithGeneric(typeName)
typeName.matches(simpleTypeRegex) -> SimpleType(typeName)
else -> throw IllegalArgumentException("Cannot define type for $typeName")
}
}
abstract fun getMainTypeName(): String
}
// CardV2
class SimpleType(typeName: String) : Type(typeName) {
override fun getMainTypeName(): String = typeName
}
// String[]
class ArrayType(typeName: String): Type(typeName) {
override fun getMainTypeName(): String = typeName.dropLast(2)
}
// BaseResponse<T>
class TypeWithGeneric(typeName: String): Type(typeName) {
override fun getMainTypeName(): String = typeName
.substringBefore("<")
fun getGenericTypeName(): String = typeName
.substringAfter("<")
.dropLast(1)
}
// Subscription<String, Int>
class MapType(typeName: String): Type(typeName) {
override fun getMainTypeName(): String = typeName.substringBefore("<")
fun getSubTypeNames(): Set<String> = typeName
.substringAfter("<")
.dropLast(1)
.split(",")
.map { it.trim() }
.toSet()
}

View File

@ -1,5 +1,7 @@
package static_analysis.utils package static_analysis.utils
import apigen.ApiGeneratorExtension
import apigen.ApiGeneratorPlugin
import org.gradle.api.Project import org.gradle.api.Project
import org.gradle.api.file.FileTree import org.gradle.api.file.FileTree
import java.io.File import java.io.File
@ -25,3 +27,8 @@ fun Project.getSources(excludes: String): FileTree = files(
.filter { it.exists() && it.isDirectory } .filter { it.exists() && it.isDirectory }
.map { it.path } .map { it.path }
).asFileTree ).asFileTree
fun Project.getApiGeneratorExtension(): ApiGeneratorExtension =
extensions.getByName(ApiGeneratorPlugin.API_GENERATOR_EXT_NAME) as ApiGeneratorExtension
fun Project.getProjectName() = rootDir.absolutePath.substringAfterLast(File.separator)

View File

@ -2,6 +2,7 @@ package static_analysis.utils
import java.io.File import java.io.File
import java.io.IOException import java.io.IOException
import java.nio.file.Path
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
fun String.runCommand( fun String.runCommand(
@ -23,3 +24,5 @@ fun String.runCommand(
null null
} }
} }
fun String.toPath(): Path = Path.of(this)

View File

@ -2,11 +2,11 @@ import groovy.json.JsonSlurper
import groovy.xml.MarkupBuilder import groovy.xml.MarkupBuilder
tasks.register('stringGenerator') { tasks.register('stringGenerator') {
Map<String, String> sources = android.languageMap def sources = android.languageMap
outputs.files(sources) outputs.files(sources)
doFirst { doLast {
generate(sources, project) generate(sources, project)
println("Strings generated!") println("Strings generated!")
} }