Skip to content

Commit a394056

Browse files
authored
fix media upload resource (#14)
* fix media upload resource, update aiplatform specs, remove tests for sttp3 * update scala name mapping
1 parent 04da004 commit a394056

File tree

5 files changed

+63
-50
lines changed

5 files changed

+63
-50
lines changed

README.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
Generates client code from Google's [disovery document](https://developers.google.com/discovery/v1/using) for your Scala (3) tech stack.
88
Currently it provides following configurations for generated code:
9-
- Http sources: [Sttp4](https://sttp.softwaremill.com/en/latest), [Sttp3](https://sttp.softwaremill.com/en/stable)
9+
- Http sources: [Sttp4](https://sttp.softwaremill.com/en/latest)
1010
- JSON codecs: [Jsoniter](https://github.com/plokhotnyuk/jsoniter-scala), [ZioJson](https://zio.dev/zio-json)
1111
- JSON Array collection type: `List`, `Vector`, `Array`, `ZioChunk`
1212

@@ -24,8 +24,8 @@ The generator can be used with any tool that can perform system calls to a comma
2424
See example under [example/generate.scala](./example/generate.scala).
2525

2626
```scala
27-
//> using scala 3.7.0
28-
//> using dep dev.rolang::gcp-codegen::0.0.5
27+
//> using scala 3.7.1
28+
//> using dep dev.rolang::gcp-codegen::0.0.7
2929

3030
import gcp.codegen.*, java.nio.file.*, GeneratorConfig.*
3131

@@ -59,7 +59,7 @@ See output in `example/out`.
5959
| -specs | Can be `stdin` or a path to the JSON file. | | |
6060
| -out-dir | Ouput directory | | |
6161
| -out-pkg | Output package | | |
62-
| -http-source | Generated http source. | [Sttp4](https://sttp.softwaremill.com/en/latest), [Sttp3](https://sttp.softwaremill.com/en/stable) | |
62+
| -http-source | Generated http source. | [Sttp4](https://sttp.softwaremill.com/en/stable) | |
6363
| -json-codec | Generated JSON codec | [Jsoniter](https://github.com/plokhotnyuk/jsoniter-scala), [ZioJson](https://zio.dev/zio-json) | |
6464
| -array-type | Collection type for JSON arrays | `List`, `Vector`, `Array`, `ZioChunk` | `List` |
6565
| -include-resources | Optional resource filter. | | |

build.sbt

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ ThisBuild / description := "Google Cloud client code generator"
22
ThisBuild / organization := "dev.rolang"
33
ThisBuild / licenses := Seq(License.MIT)
44
ThisBuild / homepage := Some(url("https://github.com/rolang/google-rest-api-codegen"))
5-
ThisBuild / scalaVersion := "3.3.6"
5+
ThisBuild / scalaVersion := "3.7.1"
66
ThisBuild / version ~= { v => if (v.contains('+')) s"${v.replace('+', '-')}-SNAPSHOT" else v }
77
ThisBuild / versionScheme := Some("early-semver")
88
ThisBuild / scmInfo := Some(
@@ -37,9 +37,7 @@ lazy val noPublish = Seq(
3737
publish / skip := true
3838
)
3939

40-
lazy val sttpClient4Version = "4.0.0-RC1"
41-
42-
lazy val sttpClient3Version = "3.10.3"
40+
lazy val sttpClient4Version = "4.0.9"
4341

4442
lazy val zioVersion = "2.1.16"
4543

@@ -63,10 +61,7 @@ lazy val root = (project in file("."))
6361
// for supporting code inspection / testing of generated code via test_gen.sh script
6462
lazy val testLocal = (project in file("test-local"))
6563
.settings(
66-
libraryDependencies ++= (
67-
dependencyByConfig("Sttp4", "Jsoniter", "ZioChunk")
68-
++ dependencyByConfig("Sttp3", "ZioJson", "List")
69-
)
64+
libraryDependencies ++= dependencyByConfig("Sttp4", "Jsoniter", "ZioChunk")
7065
)
7166
.settings(noPublish)
7267

@@ -97,7 +92,6 @@ lazy val cli = project
9792

9893
def dependencyByConfig(httpSource: String, jsonCodec: String, arrayType: String): Seq[ModuleID] = {
9994
(httpSource match {
100-
case "Sttp3" => Seq("com.softwaremill.sttp.client3" %% "core" % sttpClient3Version)
10195
case "Sttp4" => Seq("com.softwaremill.sttp.client4" %% "core" % sttpClient4Version)
10296
case other => throw new InterruptedException(s"Invalid http source: $other")
10397
}) ++ (jsonCodec match {
@@ -125,7 +119,7 @@ lazy val testProjects: CompositeProject = new CompositeProject {
125119
"iamcredentials" -> "v1",
126120
"redis" -> "v1"
127121
)
128-
httpSource <- Seq("Sttp4", "Sttp3")
122+
httpSource <- Seq("Sttp4")
129123
jsonCodec <- Seq("ZioJson", "Jsoniter")
130124
arrayType <- Seq("ZioChunk", "List")
131125
id = s"test-$apiName-$apiVersion-${httpSource}-${jsonCodec}-${arrayType}".toLowerCase()
@@ -180,8 +174,6 @@ lazy val cliBinFile: File = {
180174
lazy val buildCliBinary = taskKey[File]("")
181175
buildCliBinary := {
182176
val built = (cli / Compile / nativeLinkReleaseFast).value
183-
val destZip = new File(s"${cliBinFile.getPath()}.zip")
184-
185177
IO.copyFile(built, cliBinFile)
186178
cliBinFile
187179
}
@@ -239,8 +231,17 @@ def codegenTask(
239231
val files = listFilesRec(List(outDir), Nil)
240232

241233
// formatting (may need to find another way...)
242-
logger.info(s"Formatting sources in $outPathRel...")
243-
s"scala-cli fmt --scalafmt-conf=./.scalafmt.conf $outDir" ! ProcessLogger(_ => ()) // add logs when needed
234+
val fmtCmd = s"scala-cli fmt --scalafmt-conf=./.scalafmt.conf ${outDir.absolutePath}"
235+
logger.info(s"Formatting with '$fmtCmd'")
236+
val fmtErrs = scala.collection.mutable.ListBuffer.empty[String]
237+
fmtCmd ! ProcessLogger(
238+
_ => (),
239+
e => fmtErrs += e
240+
) match {
241+
case 0 => ()
242+
case c => throw new InterruptedException(s"Failure on code formatting: ${errs.mkString("\n")}")
243+
}
244+
244245
IO.delete(outDir / ".scala-build")
245246
logger.success(s"Formatting sources in $outPathRel done")
246247

modules/cli/src/main/scala/cli.scala

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
// for test runs using scala-cli
22
//> using jvm system
3-
//> using scala 3.6.4
3+
//> using scala 3.7.1
44
//> using file ../../../../core/shared/src/main/scala/codegen.scala
5-
//> using dep com.lihaoyi::upickle:4.1.0
5+
//> using dep com.lihaoyi::upickle:4.2.1
66

77
package gcp.codegen.cli
88

@@ -13,24 +13,21 @@ import GeneratorConfig.*
1313
import scala.concurrent.Await
1414
import scala.concurrent.ExecutionContext.Implicits.global
1515
import scala.concurrent.duration.*
16-
import scala.util.{Failure, Success}
1716

1817
@main def run(args: String*) =
1918
argsToTask(args) match
2019
case Left(err) =>
2120
Console.err.println(s"Invalid arguments: $err")
2221
sys.exit(1)
2322
case Right(task) =>
24-
Await
25-
.ready(task.run, 30.seconds)
26-
.onComplete {
27-
case Failure(exception) =>
28-
Console.err.println(s"Failure: ${exception.printStackTrace()}")
29-
sys.exit(1)
30-
case Success(files) =>
31-
println(s"Generated ${files.length} files")
32-
sys.exit(0)
33-
}
23+
try
24+
val files = Await.result(task.run, 30.seconds)
25+
println(s"Generated ${files.length} files")
26+
sys.exit(0)
27+
catch
28+
case err: Throwable =>
29+
Console.err.println(s"Failure: ${err.getMessage()}")
30+
sys.exit(1)
3431

3532
private def argsToTask(args: Seq[String]): Either[String, Task] =
3633
val argsMap = args.toList

modules/core/shared/src/main/scala/codegen.scala

Lines changed: 33 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -235,11 +235,9 @@ def generateBySpec(
235235
pkg = resourceKey.pkgName(resourcesPkg),
236236
resourcesPkg = resourcesPkg,
237237
schemasPkg = schemasPkg,
238-
baseUrl = specs.baseUrl,
239238
resourceName = resourceName,
240239
resource = resource,
241240
httpSource = config.httpSource,
242-
jsonCodec = config.jsonCodec,
243241
hasProps = p => specs.hasProps(p),
244242
arrType = config.arrayType,
245243
commonQueryParams = specs.queryParameters
@@ -304,19 +302,22 @@ def generateBySpec(
304302
val scalaKeyWords = Set("type", "import", "val", "object", "enum", "export")
305303

306304
def toScalaName(n: String): String =
307-
if scalaKeyWords.contains(n) then s"`$n`"
308-
else n.replaceAll("[^a-zA-Z0-9_]", "")
305+
n match
306+
// to fix a compiler warning like
307+
// import looks like a language import, but refers to something else: object language in object GoogleCloudAiplatformV1ExecutableCode
308+
case "language" => "Language"
309+
case _ =>
310+
if scalaKeyWords.contains(n) then s"`$n`"
311+
else n.replaceAll("[^a-zA-Z0-9_]", "")
309312

310313
def resourceCode(
311314
rootPkg: String,
312315
pkg: String,
313316
resourcesPkg: String,
314317
schemasPkg: String,
315-
baseUrl: String,
316318
resourceName: String,
317319
resource: Resource,
318320
httpSource: HttpSource,
319-
jsonCodec: JsonCodec,
320321
arrType: ArrayType,
321322
hasProps: SchemaPath => Boolean,
322323
commonQueryParams: Map[String, Parameter]
@@ -335,7 +336,7 @@ def resourceCode(
335336
"",
336337
s"object ${resourceName} {" +
337338
resource.methods
338-
.map { (k, v) =>
339+
.map { (k, method) =>
339340
def pathSegments(urlPath: String) =
340341
urlPath
341342
.split("/")
@@ -350,11 +351,11 @@ def resourceCode(
350351
) + "\")"
351352
)
352353

353-
val req = v.mediaUpload match
354-
case None => v.request.filter(_.schemaPath.forall(hasProps))
354+
val req = method.mediaUploads match
355+
case None => method.request.filter(_.schemaPath.forall(hasProps))
355356
case Some(_) => None
356357

357-
val uploadProtocol = v.mediaUpload match
358+
val uploadProtocol = method.mediaUploads match
358359
case Some(m) =>
359360
Some {
360361
val protocols = m.protocols.keySet.toList.sortBy {
@@ -366,7 +367,7 @@ def resourceCode(
366367
}
367368
case None => None
368369

369-
val (requiredParams, optParams) = v.scalaParameters.partition(_._2.required)
370+
val (requiredParams, optParams) = method.scalaParameters.partition(_._2.required)
370371
val params =
371372
requiredParams.map((n, t) => s"${toComment(t.description)}$n: ${t.scalaType(arrType)}") :::
372373
req.toList.map(r => s"request: ${r.scalaType(arrType)}") :::
@@ -375,17 +376,17 @@ def resourceCode(
375376
List(
376377
s"endpointUrl: Uri = $rootPkg.baseUrl",
377378
"commonQueryParams: QueryParameters = " + ((
378-
v.mediaUpload,
379+
method.mediaUploads,
379380
commonQueryParams.collectFirst { case ("uploadType", Parameter(_, _, e: SchemaType.Enum, _, _)) => e }
380381
) match {
381382
case (Some(m), Some(ut)) => s"""QueryParameters(uploadType = Some("${ut.values.head.value}"))"""
382383
case _ => "QueryParameters.empty"
383384
})
384385
)
385386

386-
val setReqUri = v.mediaUpload match
387+
val setReqUri = method.mediaUploads match
387388
case None =>
388-
s"val requestUri = endpointUrl.addPathSegments(List(${pathSegments(v.urlPath).mkString(", ")}))"
389+
s"val requestUri = endpointUrl.addPathSegments(List(${pathSegments(method.urlPath).mkString(", ")}))"
389390
case Some(m) =>
390391
List(
391392
"val requestUri = uploadProtocol match {",
@@ -402,7 +403,7 @@ def resourceCode(
402403
case Some(_) => """.body(request.toJsonString)"""
403404

404405
val queryParams = "\n val params = " +
405-
(v.scalaQueryParams match
406+
(method.scalaQueryParams match
406407
case Nil => "commonQueryParams.value"
407408
case qParams =>
408409
qParams
@@ -420,7 +421,7 @@ def resourceCode(
420421
case HttpSource.Sttp3 =>
421422
s"RequestT[Identity, Either[ResponseException[String, Exception], $t], Any]"
422423

423-
val (resType, mapResponse) = v.response match
424+
val (resType, mapResponse) = method.response match
424425
case Some(r) if r.schemaPath.forall(hasProps) =>
425426
val bodyType = r.scalaType(arrType)
426427

@@ -432,7 +433,7 @@ def resourceCode(
432433

433434
s"""|def ${toScalaName(k)}(\n${params.mkString(",\n")}): $resType = {$queryParams
434435
| $setReqUri
435-
| resourceRequest.${v.httpMethod.toLowerCase()}(requestUri.addParams(params))$body$mapResponse
436+
| resourceRequest.${method.httpMethod.toLowerCase()}(requestUri.addParams(params))$body$mapResponse
436437
|}""".stripMargin
437438
}
438439
.mkString("\n", "\n\n", "\n") +
@@ -584,7 +585,7 @@ case class Method(
584585
parameterOrder: List[String],
585586
response: Option[SchemaType],
586587
request: Option[SchemaType] = None,
587-
mediaUpload: Option[MediaUpload] = None
588+
private val mediaUpload: Option[MediaUpload] = None
588589
) {
589590
private lazy val flatPathParams: List[(String, Parameter)] = flatPath.toList.flatMap(p =>
590591
p.params.zipWithIndex.map((param, idx) =>
@@ -607,6 +608,20 @@ case class Method(
607608

608609
def urlPath: String = flatPath.map(_.path).getOrElse(path)
609610

611+
def mediaUploads: Option[MediaUpload] = mediaUpload.map(m =>
612+
m.copy(protocols =
613+
m.protocols.view
614+
.mapValues(p =>
615+
// map the path to flatPath on placeholders with pattern like {+var_name} if the same is found in method path
616+
// need a better solution for this
617+
"\\{(\\+.*?)\\}".r.findAllIn(p.path).toList match
618+
case v :: Nil if path.contains(v) => p.copy(path = flatPath.map(_.path).getOrElse(p.path))
619+
case _ => p
620+
)
621+
.toMap
622+
)
623+
)
624+
610625
// filter out path params if flatPath params are given
611626
private lazy val pathParams: List[(String, Parameter)] =
612627
parameters.toList.filterNot((_, p) => flatPathParams.nonEmpty && p.location == "path")

test_gen.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
# runs code generator via scala-cli for given config
55
# will ouput code under test-local/src/main/scala
66

7-
spec=pubsub
7+
spec=aiplatform
88
json_codec=jsoniter
99
http_source=sttp4
1010
array_type=list

0 commit comments

Comments
 (0)