mirror of https://github.com/go-gitea/gitea.git
Validate migration files (#18203)
JSON Schema validation for data used by Gitea during migrations Discussion at https://forum.forgefriends.org/t/common-json-schema-for-repository-information/563 Co-authored-by: Loïc Dachary <loic@dachary.org>pull/18414/head
Aravinth Manivannan
3 years ago
committed by
GitHub
parent
49dd906753
commit
3bb028cc46
@ -0,0 +1,112 @@
|
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migration
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v5"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
// Load project data from file, with optional validation
|
||||
func Load(filename string, data interface{}, validation bool) error {
|
||||
isJSON := strings.HasSuffix(filename, ".json")
|
||||
|
||||
bs, err := os.ReadFile(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if validation {
|
||||
err := validate(bs, data, isJSON)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return unmarshal(bs, data, isJSON)
|
||||
}
|
||||
|
||||
func unmarshal(bs []byte, data interface{}, isJSON bool) error {
|
||||
if isJSON {
|
||||
return json.Unmarshal(bs, data)
|
||||
}
|
||||
return yaml.Unmarshal(bs, data)
|
||||
}
|
||||
|
||||
func getSchema(filename string) (*jsonschema.Schema, error) {
|
||||
c := jsonschema.NewCompiler()
|
||||
c.LoadURL = openSchema
|
||||
return c.Compile(filename)
|
||||
}
|
||||
|
||||
func validate(bs []byte, datatype interface{}, isJSON bool) error {
|
||||
var v interface{}
|
||||
err := unmarshal(bs, &v, isJSON)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !isJSON {
|
||||
v, err = toStringKeys(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
var schemaFilename string
|
||||
switch datatype := datatype.(type) {
|
||||
case *[]*Issue:
|
||||
schemaFilename = "issue.json"
|
||||
case *[]*Milestone:
|
||||
schemaFilename = "milestone.json"
|
||||
default:
|
||||
return fmt.Errorf("file_format:validate: %T has not a validation implemented", datatype)
|
||||
}
|
||||
|
||||
sch, err := getSchema(schemaFilename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = sch.Validate(v)
|
||||
if err != nil {
|
||||
log.Error("migration validation with %s failed for\n%s", schemaFilename, string(bs))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func toStringKeys(val interface{}) (interface{}, error) {
|
||||
var err error
|
||||
switch val := val.(type) {
|
||||
case map[interface{}]interface{}:
|
||||
m := make(map[string]interface{})
|
||||
for k, v := range val {
|
||||
k, ok := k.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("found non-string key %T %s", k, k)
|
||||
}
|
||||
m[k], err = toStringKeys(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return m, nil
|
||||
case []interface{}:
|
||||
l := make([]interface{}, len(val))
|
||||
for i, v := range val {
|
||||
l[i], err = toStringKeys(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return l, nil
|
||||
default:
|
||||
return val, nil
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,39 @@
|
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migration
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v5"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestMigrationJSON_IssueOK(t *testing.T) {
|
||||
issues := make([]*Issue, 0, 10)
|
||||
err := Load("file_format_testdata/issue_a.json", &issues, true)
|
||||
assert.NoError(t, err)
|
||||
err = Load("file_format_testdata/issue_a.yml", &issues, true)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestMigrationJSON_IssueFail(t *testing.T) {
|
||||
issues := make([]*Issue, 0, 10)
|
||||
err := Load("file_format_testdata/issue_b.json", &issues, true)
|
||||
if _, ok := err.(*jsonschema.ValidationError); ok {
|
||||
errors := strings.Split(err.(*jsonschema.ValidationError).GoString(), "\n")
|
||||
assert.Contains(t, errors[1], "missing properties")
|
||||
assert.Contains(t, errors[1], "poster_id")
|
||||
} else {
|
||||
t.Fatalf("got: type %T with value %s, want: *jsonschema.ValidationError", err, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrationJSON_MilestoneOK(t *testing.T) {
|
||||
milestones := make([]*Milestone, 0, 10)
|
||||
err := Load("file_format_testdata/milestones.json", &milestones, true)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
@ -0,0 +1,14 @@
|
||||
[
|
||||
{
|
||||
"number": 1,
|
||||
"poster_id": 1,
|
||||
"poster_name": "name_a",
|
||||
"title": "title_a",
|
||||
"content": "content_a",
|
||||
"state": "closed",
|
||||
"is_locked": false,
|
||||
"created": "1985-04-12T23:20:50.52Z",
|
||||
"updated": "1986-04-12T23:20:50.52Z",
|
||||
"closed": "1987-04-12T23:20:50.52Z"
|
||||
}
|
||||
]
|
||||
@ -0,0 +1,10 @@
|
||||
- number: 1
|
||||
poster_id: 1
|
||||
poster_name: name_a
|
||||
title: title_a
|
||||
content: content_a
|
||||
state: closed
|
||||
is_locked: false
|
||||
created: 2021-05-27T15:24:13+02:00
|
||||
updated: 2021-11-11T10:52:45+01:00
|
||||
closed: 2021-11-11T10:52:45+01:00
|
||||
@ -0,0 +1,5 @@
|
||||
[
|
||||
{
|
||||
"number": 1
|
||||
}
|
||||
]
|
||||
@ -0,0 +1,20 @@
|
||||
[
|
||||
{
|
||||
"title": "title_a",
|
||||
"description": "description_a",
|
||||
"deadline": "1988-04-12T23:20:50.52Z",
|
||||
"created": "1985-04-12T23:20:50.52Z",
|
||||
"updated": "1986-04-12T23:20:50.52Z",
|
||||
"closed": "1987-04-12T23:20:50.52Z",
|
||||
"state": "closed"
|
||||
},
|
||||
{
|
||||
"title": "title_b",
|
||||
"description": "description_b",
|
||||
"deadline": "1998-04-12T23:20:50.52Z",
|
||||
"created": "1995-04-12T23:20:50.52Z",
|
||||
"updated": "1996-04-12T23:20:50.52Z",
|
||||
"closed": null,
|
||||
"state": "open"
|
||||
}
|
||||
]
|
||||
@ -0,0 +1,114 @@
|
||||
{
|
||||
"title": "Issue",
|
||||
"description": "Issues associated to a repository within a forge (Gitea, GitLab, etc.).",
|
||||
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"number": {
|
||||
"description": "Unique identifier, relative to the repository.",
|
||||
"type": "number"
|
||||
},
|
||||
"poster_id": {
|
||||
"description": "Unique identifier of the user who authored the issue.",
|
||||
"type": "number"
|
||||
},
|
||||
"poster_name": {
|
||||
"description": "Name of the user who authored the issue.",
|
||||
"type": "string"
|
||||
},
|
||||
"poster_email": {
|
||||
"description": "Email of the user who authored the issue.",
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"description": "Short description displayed as the title.",
|
||||
"type": "string"
|
||||
},
|
||||
"content": {
|
||||
"description": "Long, multiline, description.",
|
||||
"type": "string"
|
||||
},
|
||||
"ref": {
|
||||
"description": "Target branch in the repository.",
|
||||
"type": "string"
|
||||
},
|
||||
"milestone": {
|
||||
"description": "Name of the milestone.",
|
||||
"type": "string"
|
||||
},
|
||||
"state": {
|
||||
"description": "A 'closed' issue will not see any activity in the future, otherwise it is 'open'.",
|
||||
"enum": [
|
||||
"closed",
|
||||
"open"
|
||||
]
|
||||
},
|
||||
"is_locked": {
|
||||
"description": "A locked issue can only be modified by privileged users.",
|
||||
"type": "boolean"
|
||||
},
|
||||
"created": {
|
||||
"description": "Creation time.",
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"updated": {
|
||||
"description": "Last update time.",
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"closed": {
|
||||
"description": "The last time 'state' changed to 'closed'.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"labels": {
|
||||
"description": "List of labels.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "label.json"
|
||||
}
|
||||
},
|
||||
"reactions": {
|
||||
"description": "List of reactions.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "reaction.json"
|
||||
}
|
||||
},
|
||||
"assignees": {
|
||||
"description": "List of assignees.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"description": "Name of a user assigned to the issue.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"number",
|
||||
"poster_id",
|
||||
"poster_name",
|
||||
"title",
|
||||
"content",
|
||||
"state",
|
||||
"is_locked",
|
||||
"created",
|
||||
"updated"
|
||||
]
|
||||
},
|
||||
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$id": "http://example.com/issue.json",
|
||||
"$$target": "issue.json"
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
{
|
||||
"title": "Label",
|
||||
"description": "Label associated to an issue.",
|
||||
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"name": {
|
||||
"description": "Name of the label, unique within the repository.",
|
||||
"type": "string"
|
||||
},
|
||||
"color": {
|
||||
"description": "Color code of the label.",
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"description": "Long, multiline, description.",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name"
|
||||
],
|
||||
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$id": "label.json",
|
||||
"$$target": "label.json"
|
||||
}
|
||||
@ -0,0 +1,67 @@
|
||||
{
|
||||
"title": "Milestone",
|
||||
"description": "Milestone associated to a repository within a forge.",
|
||||
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"title": {
|
||||
"description": "Short description.",
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"description": "Long, multiline, description.",
|
||||
"type": "string"
|
||||
},
|
||||
"deadline": {
|
||||
"description": "Deadline after which the milestone is overdue.",
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"created": {
|
||||
"description": "Creation time.",
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"updated": {
|
||||
"description": "Last update time.",
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"closed": {
|
||||
"description": "The last time 'state' changed to 'closed'.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"state": {
|
||||
"description": "A 'closed' issue will not see any activity in the future, otherwise it is 'open'.",
|
||||
"enum": [
|
||||
"closed",
|
||||
"open"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"title",
|
||||
"description",
|
||||
"deadline",
|
||||
"created",
|
||||
"updated",
|
||||
"closed",
|
||||
"state"
|
||||
]
|
||||
},
|
||||
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$id": "http://example.com/milestone.json",
|
||||
"$$target": "milestone.json"
|
||||
}
|
||||
@ -0,0 +1,29 @@
|
||||
{
|
||||
"title": "Reaction",
|
||||
"description": "Reaction associated to an issue or a comment.",
|
||||
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"user_id": {
|
||||
"description": "Unique identifier of the user who authored the reaction.",
|
||||
"type": "number"
|
||||
},
|
||||
"user_name": {
|
||||
"description": "Name of the user who authored the reaction.",
|
||||
"type": "string"
|
||||
},
|
||||
"content": {
|
||||
"description": "Representation of the reaction",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"user_id",
|
||||
"content"
|
||||
],
|
||||
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$id": "http://example.com/reaction.json",
|
||||
"$$target": "reaction.json"
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build bindata
|
||||
// +build bindata
|
||||
|
||||
package migration
|
||||
|
||||
//go:generate go run ../../build/generate-bindata.go ../../modules/migration/schemas migration bindata.go
|
||||
@ -0,0 +1,40 @@
|
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build !bindata
|
||||
// +build !bindata
|
||||
|
||||
package migration
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
func openSchema(s string) (io.ReadCloser, error) {
|
||||
u, err := url.Parse(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
basename := path.Base(u.Path)
|
||||
filename := basename
|
||||
//
|
||||
// Schema reference each other within the schemas directory but
|
||||
// the tests run in the parent directory.
|
||||
//
|
||||
if _, err := os.Stat(filename); os.IsNotExist(err) {
|
||||
filename = filepath.Join("schemas", basename)
|
||||
//
|
||||
// Integration tests run from the git root directory, not the
|
||||
// directory in which the test source is located.
|
||||
//
|
||||
if _, err := os.Stat(filename); os.IsNotExist(err) {
|
||||
filename = filepath.Join("modules/migration/schemas", basename)
|
||||
}
|
||||
}
|
||||
return os.Open(filename)
|
||||
}
|
||||
@ -0,0 +1,17 @@
|
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build bindata
|
||||
// +build bindata
|
||||
|
||||
package migration
|
||||
|
||||
import (
|
||||
"io"
|
||||
"path"
|
||||
)
|
||||
|
||||
func openSchema(filename string) (io.ReadCloser, error) {
|
||||
return Assets.Open(path.Base(filename))
|
||||
}
|
||||
Loading…
Reference in New Issue