Merge branch 'master' into feature/upgrade-docker-deployment-flow

package.json

@@ -43,13 +43,14 @@
     "node-fetch": "^2.6.0",
     "node-print": "0.0.4",
     "nodemailer": "^6.2.1",
+    "node-schedule": "^1.3.2",
     "notevil": "^1.3.1",
     "path-to-regexp": "^3.0.0",
     "redis": "^2.8.0",
     "reflect-metadata": "^0.1.13",
     "request": "^2.88.0",
     "request-promise": "^4.2.4",
-    "sequelize": "^5.8.7",
+    "sequelize": "^5.10.1",
     "sequelize-typescript": "^1.0.0-beta.3",
     "svg-captcha": "^1.4.0",
     "underscore": "^1.9.1",
@@ -70,9 +71,11 @@
     "@types/mockjs": "^1.0.2",
     "@types/node": "^12.0.3",
     "@types/nodemailer": "^6.2.0",
+    "@types/node-schedule": "^1.2.3",
     "@types/redis": "^2.8.13",
     "@types/request": "^2.48.1",
     "@types/request-promise": "^4.1.44",
+    "@types/sequelize": "^4.28.4",
     "@types/underscore": "^1.8.18",
     "babel-eslint": "^10.0.1",
     "chai": "^4.2.0",

src/config/config.dev.ts

@@ -1,10 +1,10 @@
 import { IConfigOptions } from "../types"
 
 let config: IConfigOptions = {
-  version: '2.3',
+  version: 'v2.1.0',
   serve: {
     port: 8080,
-    path: "", // 服务context path
+    path: '',
   },
   keys: ['some secret hurr'],
   session: {

src/config/config.local.ts

@@ -4,7 +4,7 @@ let config: IConfigOptions = {
   version: '2.3',
   serve: {
     port: 8080,
-    path: "", // 服务context path
+    path: '',
   },
   keys: ['some secret hurr'],
   session: {

src/config/config.prod.ts

@@ -1,11 +1,11 @@
 import { IConfigOptions } from "../types"
 
 // 先从环境变量取配置
-let config: IConfigOptions = {
+let config: IConfigOptions =  {
     version: '2.3',
     serve: {
         port: (process.env.EXPOSE_PORT && parseInt(process.env.EXPOSE_PORT)) || 8080,
-        path: process.env.EXPOSE_PATH || "" , // 服务context path
+        path: '',
     },
     keys: ['some secret hurr'],
     session: {

src/routes/account.ts

@@ -7,6 +7,9 @@ import { QueryInclude } from '../models'
 import { Op } from 'sequelize'
 import MailService from '../service/mail'
 import * as md5 from 'md5'
+import { isLoggedIn } from './base'
+import { AccessUtils } from './utils/access'
+import { COMMON_ERROR_RES } from './utils/const'
 
 
 
@@ -36,7 +39,11 @@ router.get('/account/count', async (ctx) => {
   }
 })
 
-router.get('/account/list', async (ctx) => {
+router.get('/account/list', isLoggedIn, async (ctx) => {
+  // if (!AccessUtils.isAdmin(ctx.session.id)) {
+  //   ctx.body = COMMON_ERROR_RES.ACCESS_DENY
+  //   return
+  // }
   let where = {}
   let { name } = ctx.query
   if (name) {
@@ -172,7 +179,11 @@ router.post('/account/update', async (ctx) => {
   }
 })
 
-router.get('/account/remove', async (ctx) => {
+router.get('/account/remove', isLoggedIn, async (ctx) => {
+  if (!AccessUtils.isAdmin(ctx.session.id)) {
+    ctx.body = COMMON_ERROR_RES.ACCESS_DENY
+    return
+  }
   if (process.env.TEST_MODE === 'true') {
     ctx.body = {
       data: await User.destroy({

src/routes/analytics.ts

@@ -6,10 +6,11 @@ const moment = require('moment')
 const Sequelize = require('sequelize')
 const SELECT = { type: Sequelize.QueryTypes.SELECT }
 import sequelize from '../models/sequelize'
+import { isLoggedIn } from './base'
 const YYYY_MM_DD = 'YYYY-MM-DD'
 
 // 最近 30 天新建仓库数
-router.get('/app/analytics/repositories/created', async (ctx) => {
+router.get('/app/analytics/repositories/created', isLoggedIn, async (ctx) => {
   let start = moment().startOf('day').subtract(30, 'days').format(YYYY_MM_DD)
   let end = moment().startOf('day').format(YYYY_MM_DD)
   let sql = `
@@ -34,7 +35,7 @@ router.get('/app/analytics/repositories/created', async (ctx) => {
 })
 
 // 最近 30 天活跃仓库数
-router.get('/app/analytics/repositories/updated', async (ctx) => {
+router.get('/app/analytics/repositories/updated', isLoggedIn, async (ctx) => {
   let start = moment().startOf('day').subtract(30, 'days').format(YYYY_MM_DD)
   let end = moment().startOf('day').format(YYYY_MM_DD)
   let sql = `
@@ -59,7 +60,7 @@ router.get('/app/analytics/repositories/updated', async (ctx) => {
 })
 
 // 最近 30 天活跃用户
-router.get('/app/analytics/users/activation', async (ctx) => {
+router.get('/app/analytics/users/activation', isLoggedIn, async (ctx) => {
   let start = moment().startOf('day').subtract(30, 'days').format(YYYY_MM_DD)
   let end = moment().startOf('day').format(YYYY_MM_DD)
   let sql = `
@@ -84,7 +85,7 @@ router.get('/app/analytics/users/activation', async (ctx) => {
 })
 
 // 最近 30 天活跃仓库
-router.get('/app/analytics/repositories/activation', async (ctx) => {
+router.get('/app/analytics/repositories/activation', isLoggedIn, async (ctx) => {
   let start = moment().startOf('day').subtract(30, 'days').format(YYYY_MM_DD)
   let end = moment().startOf('day').format(YYYY_MM_DD)
   let sql = `

src/routes/base.ts

@@ -0,0 +1,15 @@
+import * as _ from 'lodash'
+import { ParameterizedContext } from 'koa'
+const inTestMode = process.env.TEST_MODE === 'true'
+
+
+export async function isLoggedIn(ctx: ParameterizedContext<any, any>, next: () => Promise<any>) {
+  if (!inTestMode && (!ctx.session || ctx.session.id == undefined)) {
+    ctx.body = {
+      isOk: false,
+      errMsg: 'need login',
+    }
+  } else {
+    await next()
+  }
+}

src/routes/mock.ts

@@ -100,7 +100,6 @@ router.get('/app/plugin/:repositories', async (ctx) => {
     result.push(generatePlugin(protocol, ctx.host, repository))
   }
 
-  ctx.enco
   ctx.type = 'application/x-javascript; charset=utf-8'
   ctx.body = result.join('\n')
 })
@@ -178,26 +177,40 @@ router.all('/app/mock/:repositoryId(\\d+)/:url(.+)', async (ctx) => {
 
   // matching by params
   if (matchedItfList.length > 1) {
+    const params = {
+      ...ctx.request.query,
+      ...ctx.request.body,
+    }
+    const paramsKeysCnt = Object.keys(params).length
     matchedItfList = matchedItfList.filter(x => {
-      const params = {
-        ...ctx.request.query,
-        ...ctx.request.body,
-      }
       const parsedUrl = urlPkg.parse(x.url)
-      const pairs = parsedUrl.query.split('&').map(x => x.split('='))
+      const pairs = parsedUrl.query ? parsedUrl.query.split('&').map(x => x.split('=')) : []
+      // 接口没有定义参数时看请求是否有参数
+      if (pairs.length === 0) {
+        return paramsKeysCnt === 0
+      }
+      // 接口定义参数时看每一项的参数是否一致
       for (const p of pairs) {
         const key = p[0]
         const val = p[1]
-        if (params[key] == val) {
-          return true
+        if (params[key] != val) {
+          return false
         }
       }
-      // for (let key in query) {
-      // }
-      return false
+      return true
     })
   }
 
+  // 多个协同仓库的结果优先返回当前仓库的
+  if (matchedItfList.length > 1) {
+    const currProjMatchedItfList = matchedItfList.filter(x => x.repositoryId === repositoryId)
+    // 如果直接存在当前仓库的就当做结果集，否则放弃
+    if (currProjMatchedItfList.length > 0) {
+      matchedItfList = currProjMatchedItfList
+    }
+  }
+
+
   for (const item of matchedItfList) {
     itf = item
     let url = item.url
@@ -246,7 +259,7 @@ router.all('/app/mock/:repositoryId(\\d+)/:url(.+)', async (ctx) => {
         }
       }
     } else if (listMatched.length === 0) {
-      ctx.body = {isOk: false, errMsg: '未匹配到任何接口 No matched interface'}
+      ctx.body = { isOk: false, errMsg: '未匹配到任何接口，请检查请求类型是否一致。' }
       return
     } else {
       loadDataId = listMatched[0].id
@@ -268,7 +281,7 @@ router.all('/app/mock/:repositoryId(\\d+)/:url(.+)', async (ctx) => {
     })
     let passed = true
     let pFailed: Property | undefined
-    let params = method === 'GET' ? ctx.request.query : ctx.request.body
+    let params = method === 'GET' ? {...ctx.request.query} : {...ctx.request.body}
     // http request中head的参数未添加，会造成head中的参数必填勾选后即使header中有值也会检查不通过
     params = Object.assign(params, ctx.request.headers)
     for (const p of requiredProperties) {
@@ -283,7 +296,6 @@ router.all('/app/mock/:repositoryId(\\d+)/:url(.+)', async (ctx) => {
         isOk: false,
         errMsg: `必选参数${pFailed.name}未传值。 Required parameter ${pFailed.name} has no value.`,
       }
-      ctx.status = 500
       return
     }
   }
@@ -297,11 +309,16 @@ router.all('/app/mock/:repositoryId(\\d+)/:url(.+)', async (ctx) => {
   })
   requestProperties = requestProperties.map((item: any) => item.toJSON())
   let requestData = Tree.ArrayToTreeToTemplateToData(requestProperties)
-  Object.assign(requestData, ctx.query)
+  Object.assign(requestData, ctx.params)
   const data = Tree.ArrayToTreeToTemplateToData(properties, requestData)
   ctx.type = 'json'
   ctx.status = itf.status
   ctx.body = JSON.stringify(data, undefined, 2)
+  const Location = data.Location
+  if (Location && itf.status === 301) {
+    ctx.redirect(Location)
+    return
+  }
   if (itf && itf.url.indexOf('[callback]=') > -1) {
     const query = querystring.parse(itf.url.substring(itf.url.indexOf('?') + 1))
     const cbName = query['[callback]']

src/routes/organization.ts

@@ -5,6 +5,9 @@ import * as _ from 'lodash'
 import Pagination from './utils/pagination'
 import OrganizationService from '../service/organization'
 import { Op, FindOptions }  from 'sequelize'
+import { isLoggedIn } from './base'
+import { AccessUtils, ACCESS_TYPE } from './utils/access'
+import { COMMON_ERROR_RES } from './utils/const'
 
 router.get('/app/get', async (ctx, next) => {
   let data: any = {}
@@ -56,16 +59,7 @@ router.get('/organization/list', async (ctx) => {
     pagination,
   }
 })
-router.get('/organization/owned', async (ctx) => {
-  if (!ctx.session.id) {
-    ctx.body = {
-      data: {
-        isOk: false,
-        errMsg: 'not login'
-      }
-    }
-    return
-  }
+router.get('/organization/owned', isLoggedIn, async (ctx) => {
   let where = {}
   let { name } = ctx.query
   if (name) {
@@ -90,16 +84,7 @@ router.get('/organization/owned', async (ctx) => {
     pagination: undefined,
   }
 })
-router.get('/organization/joined', async (ctx) => {
-  if (!ctx.session.id) {
-    ctx.body = {
-      data: {
-        isOk: false,
-        errMsg: 'not login'
-      }
-    }
-    return
-  }
+router.get('/organization/joined', isLoggedIn, async (ctx) => {
   let where = {}
   let { name } = ctx.query
   if (name) {
@@ -127,7 +112,12 @@ router.get('/organization/joined', async (ctx) => {
   }
 })
 router.get('/organization/get', async (ctx) => {
-  let organization = await Organization.findByPk(ctx.query.id, {
+  const organizationId = +ctx.query.id
+  if (!await AccessUtils.canUserAccess(ACCESS_TYPE.ORGANIZATION, ctx.session.id, organizationId)) {
+    ctx.body = COMMON_ERROR_RES.ACCESS_DENY
+    return
+  }
+  const organization = await Organization.findByPk(ctx.query.id, {
     attributes: { exclude: [] },
     include: [QueryInclude.Creator, QueryInclude.Owner, QueryInclude.Members],
   } as any)
@@ -135,7 +125,7 @@ router.get('/organization/get', async (ctx) => {
     data: organization,
   }
 })
-router.post('/organization/create', async (ctx) => {
+router.post('/organization/create', isLoggedIn, async (ctx) => {
   let creatorId = ctx.session.id
   let body = Object.assign({}, ctx.request.body, { creatorId, ownerId: creatorId })
   let created = await Organization.create(body)
@@ -151,8 +141,13 @@ router.post('/organization/create', async (ctx) => {
     data: filled,
   }
 })
-router.post('/organization/update', async (ctx, next) => {
+router.post('/organization/update', isLoggedIn, async (ctx, next) => {
   let body = Object.assign({}, ctx.request.body)
+  const organizationId = +body.id
+  if (!await AccessUtils.canUserAccess(ACCESS_TYPE.ORGANIZATION, ctx.session.id, organizationId)) {
+    ctx.body = COMMON_ERROR_RES.ACCESS_DENY
+    return
+  }
   delete body.creatorId
   // DONE 2.2 支持转移团队
   // delete body.ownerId
@@ -190,16 +185,26 @@ router.post('/organization/update', async (ctx, next) => {
     await Logger.create({ creatorId, userId, type: 'exit', organizationId: id })
   }
 })
-router.post('/organization/transfer', async (ctx) => {
+router.post('/organization/transfer',  isLoggedIn, async (ctx) => {
   let { id, ownerId } = ctx.request.body
+  const organizationId = +id
+  if (!await AccessUtils.canUserAccess(ACCESS_TYPE.ORGANIZATION, ctx.session.id, organizationId)) {
+    ctx.body = COMMON_ERROR_RES.ACCESS_DENY
+    return
+  }
   let body = { ownerId }
   let result = await Organization.update(body, { where: { id } })
   ctx.body = {
     data: result[0],
   }
 })
-router.get('/organization/remove', async (ctx, next) => {
+router.get('/organization/remove', isLoggedIn, async (ctx, next) => {
   let { id } = ctx.query
+  const organizationId = +id
+  if (!await AccessUtils.canUserAccess(ACCESS_TYPE.ORGANIZATION, ctx.session.id, organizationId)) {
+    ctx.body = COMMON_ERROR_RES.ACCESS_DENY
+    return
+  }
   let result = await Organization.destroy({ where: { id } })
   let repositories = await Repository.findAll({
     where: { organizationId: id },

src/routes/postman.ts

@@ -1,9 +1,14 @@
 import router from './router'
 import { COMMON_ERROR_RES } from './utils/const'
 import PostmanService from '../service/postman'
+import { AccessUtils, ACCESS_TYPE } from './utils/access'
 
 router.get('/postman/export', async (ctx) => {
   const repoId = +ctx.query.id
+  if (!await AccessUtils.canUserAccess(ACCESS_TYPE.REPOSITORY, ctx.session.id, repoId)) {
+    ctx.body = COMMON_ERROR_RES.ACCESS_DENY
+    return
+  }
   if (!(repoId > 0)) {
     ctx.data = COMMON_ERROR_RES.ERROR_PARAMS
   }

src/routes/repository.ts

@@ -176,26 +176,46 @@ router.get('/repository/get', async (ctx) => {
     return
   }
   const tryCache = await RedisService.getCache(CACHE_KEY.REPOSITORY_GET, ctx.query.id)
-  let repository: Repository
+  let repository: Partial<Repository>
   if (tryCache) {
     repository = JSON.parse(tryCache)
   } else {
-    repository = await Repository.findByPk(ctx.query.id, {
-      attributes: { exclude: [] },
-      include: [
-        QueryInclude.Creator,
-        QueryInclude.Owner,
-        QueryInclude.Locker,
-        QueryInclude.Members,
-        QueryInclude.Organization,
-        QueryInclude.RepositoryHierarchy,
-        QueryInclude.Collaborators
-      ],
-      order: [
-        [{ model: Module, as: 'modules' }, 'priority', 'asc'],
-        [{ model: Module, as: 'modules' }, { model: Interface, as: 'interfaces' }, 'priority', 'asc']
-      ] as any
-    })
+    // 分开查询减少
+    let [repositoryOmitModules, repositoryModules] = await Promise.all([
+      Repository.findByPk(ctx.query.id, {
+        attributes: { exclude: [] },
+        include: [
+          QueryInclude.Creator,
+          QueryInclude.Owner,
+          QueryInclude.Locker,
+          QueryInclude.Members,
+          QueryInclude.Organization,
+          QueryInclude.Collaborators,
+        ]
+      }),
+      Repository.findByPk(ctx.query.id, {
+        attributes: { exclude: [] },
+        include: [QueryInclude.RepositoryHierarchy],
+        order: [
+          [{ model: Module, as: 'modules' }, 'priority', 'asc'],
+          [
+            { model: Module, as: 'modules' },
+            { model: Interface, as: 'interfaces' },
+            'priority',
+            'asc'
+          ]
+        ]
+      })
+    ])
+    repository = {
+      ...repositoryOmitModules.toJSON(),
+      ...repositoryModules.toJSON()
+    }
+    await RedisService.setCache(
+      CACHE_KEY.REPOSITORY_GET,
+      JSON.stringify(repository),
+      ctx.query.id
+    )
     await RedisService.setCache(CACHE_KEY.REPOSITORY_GET, JSON.stringify(repository), ctx.query.id)
   }
   ctx.body = {

src/routes/router.ts

@@ -1,7 +1,7 @@
 import * as Router from 'koa-router'
 import config from '../config'
 
-let router = new Router({prefix: config.serve.path})
+ let router = new Router({prefix: config.serve.path})
 
 // index
 router.get('/', (ctx) => {

src/routes/utils/access.ts

@@ -1,15 +1,36 @@
 import OrganizationService from '../../service/organization'
 import RepositoryService from '../../service/repository'
+import { Module, Interface, Property } from '../../models'
 
-export enum ACCESS_TYPE { ORGANIZATION, REPOSITORY, USER }
+export enum ACCESS_TYPE { ORGANIZATION, REPOSITORY, MODULE, INTERFACE, PROPERTY, USER, ADMIN }
+const inTestMode = process.env.TEST_MODE === 'true'
 
 export class AccessUtils {
   public static async canUserAccess(accessType: ACCESS_TYPE, curUserId: number, entityId: number): Promise<boolean> {
+    if (inTestMode) {
+      return true
+    }
     if (accessType === ACCESS_TYPE.ORGANIZATION) {
       return await OrganizationService.canUserAccessOrganization(curUserId, entityId)
     } else if (accessType === ACCESS_TYPE.REPOSITORY) {
       return await RepositoryService.canUserAccessRepository(curUserId, entityId)
+    } else if (accessType === ACCESS_TYPE.MODULE) {
+      const mod = await Module.findByPk(entityId)
+      return await RepositoryService.canUserAccessRepository(curUserId, mod.repositoryId)
+    } else if (accessType === ACCESS_TYPE.INTERFACE) {
+      const itf = await Interface.findByPk(entityId)
+      return await RepositoryService.canUserAccessRepository(curUserId, itf.repositoryId)
+    } else if (accessType === ACCESS_TYPE.PROPERTY) {
+      const p = await Property.findByPk(entityId)
+      return await RepositoryService.canUserAccessRepository(curUserId, p.repositoryId)
     }
     return false
   }
+
+  public static isAdmin(curUserId: number) {
+    if (inTestMode) {
+      return true
+    }
+    return curUserId === 1
+  }
 }

src/routes/utils/const.ts

@@ -6,4 +6,13 @@ export const COMMON_ERROR_RES = {
   ERROR_PARAMS: { isOk: false, errMsg: '参数错误' },
   ACCESS_DENY: { isOk: false, errMsg: '您没有访问权限' },
   NOT_LOGIN: { isOk: false, errMsg: '您未登陆，或登陆状态过期。请登陆后重试' },
-}
+}
+
+export enum DATE_CONST {
+  SECOND = 1000,
+  MINUTE = 1000 * 60,
+  HOUR = 1000 * 60 * 60,
+  DAY = 1000 * 60 * 60 * 24,
+  MONTH = 1000 * 60 * 60 * 24 * 30,
+  YEAR = 1000 * 60 * 60 * 24 * 365,
+}

src/routes/utils/url.ts

@@ -29,6 +29,7 @@ export default class UrlUtils {
     let re = pathToRegexp(pattern)
     return re.test(url)
   }
+
   public static getUrlPattern = (pattern: string) => {
     pattern = UrlUtils.getRelative(pattern)
     return pathToRegexp(pattern)

src/scripts/app.ts

@@ -8,6 +8,7 @@ import * as cors from 'kcors'
 import * as bodyParser from 'koa-body'
 import router from '../routes'
 import config from '../config'
+import { startTask } from '../service/task'
 
 const app = new Koa()
 let appAny: any = app
@@ -54,4 +55,6 @@ app.use(bodyParser({ multipart: true }))
 
 app.use(router.routes())
 
+startTask()
+
 export default app

src/service/task.ts

@@ -0,0 +1,28 @@
+import * as schedule from 'node-schedule'
+import { Interface } from '../models'
+import { Op } from 'sequelize'
+import { DATE_CONST } from '../routes/utils/const'
+
+export async function startTask() {
+
+  console.log(`Starting task: locker check`)
+
+  /**
+   * 每5分钟检查lock超时
+   */
+  schedule.scheduleJob('*/5 * * * *', async () => {
+    // tslint:disable-next-line: no-null-keyword
+    const [num] = await Interface.update({ lockerId: null }, {
+      where: {
+        lockerId: {
+          [Op.gt]: 0,
+        },
+        updatedAt: {
+          [Op.lt]: new Date(Date.now() - DATE_CONST.DAY),
+        },
+      },
+    })
+
+    num > 0 && console.log(`cleared ${num} locks`)
+  })
+}

src/types/index.d.ts

@@ -14,8 +14,8 @@ declare interface RedisAndClusterOptions extends RedisOptions {
 declare interface IConfigOptions {
   version: string
   serve: {
-    port: number,
-    path: string
+    port: number
+    path: string // Context Path
   },
   keys: string[]
   session: {