{
    "componentChunkName": "component---src-templates-case-studies-ts",
    "path": "/case-studies",
    "result": {"pageContext":{"i18nData":{"customerShowcases":{"zh":{"data":[{"id":13,"attributes":{"name":"58","website":null,"createdAt":"2022-05-12T09:59:45.880Z","updatedAt":"2022-05-12T10:00:08.185Z","locale":"zh","logo":{"data":{"id":193,"attributes":{"name":"58同城.png","alternativeText":"58同城.png","caption":"58同城.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_58同城.png","hash":"thumbnail_58_b8196c9bdc","ext":".png","mime":"image/png","width":245,"height":73,"size":7.56,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_58_b8196c9bdc.png"}},"hash":"58_b8196c9bdc","ext":".png","mime":"image/png","size":9.98,"url":"https://official-media.sphere-ex.com/58_b8196c9bdc.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T09:59:34.418Z","updatedAt":"2022-05-12T09:59:34.418Z"}}},"localizations":{"data":[{"id":14,"attributes":{"name":"58","website":null,"createdAt":"2022-05-12T10:00:08.148Z","updatedAt":"2022-05-12T10:00:08.148Z","locale":"en"}}]}}},{"id":16,"attributes":{"name":"360数科","website":null,"createdAt":"2022-05-12T10:02:26.586Z","updatedAt":"2022-05-12T10:02:26.586Z","locale":"zh","logo":{"data":{"id":194,"attributes":{"name":"360 数科.png","alternativeText":"360 数科.png","caption":"360 数科.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_360 数科.png","hash":"thumbnail_360_ba73dda734","ext":".png","mime":"image/png","width":245,"height":73,"size":13.34,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_360_ba73dda734.png"}},"hash":"360_ba73dda734","ext":".png","mime":"image/png","size":24.34,"url":"https://official-media.sphere-ex.com/360_ba73dda734.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T10:02:01.076Z","updatedAt":"2022-05-12T10:02:01.076Z"}}},"localizations":{"data":[{"id":15,"attributes":{"name":"360数科","website":null,"createdAt":"2022-05-12T10:02:13.011Z","updatedAt":"2022-05-12T10:02:26.617Z","locale":"en"}}]}}},{"id":17,"attributes":{"name":"500PX","website":null,"createdAt":"2022-05-12T10:03:56.480Z","updatedAt":"2022-05-12T10:04:05.703Z","locale":"zh","logo":{"data":{"id":195,"attributes":{"name":"500px.png","alternativeText":"500px.png","caption":"500px.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_500px.png","hash":"thumbnail_500px_c2be77bdb0","ext":".png","mime":"image/png","width":245,"height":73,"size":4.56,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_500px_c2be77bdb0.png"}},"hash":"500px_c2be77bdb0","ext":".png","mime":"image/png","size":5.28,"url":"https://official-media.sphere-ex.com/500px_c2be77bdb0.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T10:03:43.369Z","updatedAt":"2022-05-12T10:03:43.369Z"}}},"localizations":{"data":[{"id":18,"attributes":{"name":"500PX","website":null,"createdAt":"2022-05-12T10:04:05.674Z","updatedAt":"2022-05-12T10:04:05.674Z","locale":"en"}}]}}},{"id":19,"attributes":{"name":"爱番番","website":null,"createdAt":"2022-05-12T10:04:52.902Z","updatedAt":"2022-05-12T10:05:05.715Z","locale":"zh","logo":{"data":{"id":196,"attributes":{"name":"爱番番.png","alternativeText":"爱番番.png","caption":"爱番番.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_爱番番.png","hash":"thumbnail__2ec44848c9","ext":".png","mime":"image/png","width":245,"height":73,"size":7.15,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__2ec44848c9.png"}},"hash":"_2ec44848c9","ext":".png","mime":"image/png","size":13.36,"url":"https://official-media.sphere-ex.com/_2ec44848c9.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T10:04:35.351Z","updatedAt":"2022-05-12T10:04:35.351Z"}}},"localizations":{"data":[{"id":20,"attributes":{"name":"爱番番","website":null,"createdAt":"2022-05-12T10:05:05.682Z","updatedAt":"2022-05-12T10:05:05.682Z","locale":"en"}}]}}},{"id":21,"attributes":{"name":"爱奇艺","website":null,"createdAt":"2022-05-12T10:05:29.353Z","updatedAt":"2022-05-12T10:05:39.064Z","locale":"zh","logo":{"data":{"id":197,"attributes":{"name":"爱奇艺.png","alternativeText":"爱奇艺.png","caption":"爱奇艺.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_爱奇艺.png","hash":"thumbnail__ff2f4ac641","ext":".png","mime":"image/png","width":245,"height":73,"size":11.2,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__ff2f4ac641.png"}},"hash":"_ff2f4ac641","ext":".png","mime":"image/png","size":19.87,"url":"https://official-media.sphere-ex.com/_ff2f4ac641.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T10:05:20.081Z","updatedAt":"2022-05-12T10:05:20.081Z"}}},"localizations":{"data":[{"id":22,"attributes":{"name":"爱奇艺","website":null,"createdAt":"2022-05-12T10:05:39.037Z","updatedAt":"2022-05-12T10:05:39.037Z","locale":"en"}}]}}},{"id":24,"attributes":{"name":"百果园","website":null,"createdAt":"2022-05-12T10:09:39.577Z","updatedAt":"2022-05-12T10:09:39.577Z","locale":"zh","logo":{"data":{"id":198,"attributes":{"name":"百果园.png","alternativeText":"百果园.png","caption":"百果园.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_百果园.png","hash":"thumbnail__b5aa05467d","ext":".png","mime":"image/png","width":245,"height":73,"size":8.48,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__b5aa05467d.png"}},"hash":"_b5aa05467d","ext":".png","mime":"image/png","size":15.35,"url":"https://official-media.sphere-ex.com/_b5aa05467d.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T10:09:15.192Z","updatedAt":"2022-05-12T10:09:15.192Z"}}},"localizations":{"data":[{"id":23,"attributes":{"name":"百果园","website":null,"createdAt":"2022-05-12T10:09:23.967Z","updatedAt":"2022-05-12T10:09:39.612Z","locale":"en"}}]}}},{"id":25,"attributes":{"name":"编程猫","website":null,"createdAt":"2022-05-12T10:10:23.647Z","updatedAt":"2022-05-12T10:10:32.569Z","locale":"zh","logo":{"data":{"id":199,"attributes":{"name":"编程猫.png","alternativeText":"编程猫.png","caption":"编程猫.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_编程猫.png","hash":"thumbnail__8a3da21399","ext":".png","mime":"image/png","width":245,"height":73,"size":6.46,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__8a3da21399.png"}},"hash":"_8a3da21399","ext":".png","mime":"image/png","size":9.01,"url":"https://official-media.sphere-ex.com/_8a3da21399.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T10:10:12.303Z","updatedAt":"2022-05-12T10:10:12.303Z"}}},"localizations":{"data":[{"id":26,"attributes":{"name":"编程猫","website":null,"createdAt":"2022-05-12T10:10:32.541Z","updatedAt":"2022-05-12T10:10:32.541Z","locale":"en"}}]}}},{"id":27,"attributes":{"name":"天威诚信","website":null,"createdAt":"2022-05-12T10:10:58.220Z","updatedAt":"2022-05-12T10:11:06.416Z","locale":"zh","logo":{"data":{"id":200,"attributes":{"name":"诚信天威.png","alternativeText":"诚信天威.png","caption":"诚信天威.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_诚信天威.png","hash":"thumbnail__bd7c360f7e","ext":".png","mime":"image/png","width":245,"height":73,"size":7.26,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__bd7c360f7e.png"}},"hash":"_bd7c360f7e","ext":".png","mime":"image/png","size":13.57,"url":"https://official-media.sphere-ex.com/_bd7c360f7e.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T10:10:47.908Z","updatedAt":"2022-05-12T10:10:47.908Z"}}},"localizations":{"data":[{"id":28,"attributes":{"name":"天威诚信","website":null,"createdAt":"2022-05-12T10:11:06.381Z","updatedAt":"2022-05-12T10:11:06.381Z","locale":"en"}}]}}},{"id":30,"attributes":{"name":"大搜车","website":null,"createdAt":"2022-05-12T11:05:07.827Z","updatedAt":"2022-05-12T11:05:07.827Z","locale":"zh","logo":{"data":{"id":201,"attributes":{"name":"大搜车.png","alternativeText":"大搜车.png","caption":"大搜车.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_大搜车.png","hash":"thumbnail__1e1776bd45","ext":".png","mime":"image/png","width":245,"height":73,"size":5.03,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__1e1776bd45.png"}},"hash":"_1e1776bd45","ext":".png","mime":"image/png","size":7.47,"url":"https://official-media.sphere-ex.com/_1e1776bd45.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:04:51.089Z","updatedAt":"2022-05-12T11:04:51.089Z"}}},"localizations":{"data":[{"id":29,"attributes":{"name":"大搜车","website":null,"createdAt":"2022-05-12T11:05:00.232Z","updatedAt":"2022-05-12T11:05:07.849Z","locale":"en"}}]}}},{"id":31,"attributes":{"name":"当当","website":null,"createdAt":"2022-05-12T11:05:35.672Z","updatedAt":"2022-05-12T11:05:45.516Z","locale":"zh","logo":{"data":{"id":202,"attributes":{"name":"当当.png","alternativeText":"当当.png","caption":"当当.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_当当.png","hash":"thumbnail__454b261869","ext":".png","mime":"image/png","width":245,"height":73,"size":9.01,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__454b261869.png"}},"hash":"_454b261869","ext":".png","mime":"image/png","size":15.73,"url":"https://official-media.sphere-ex.com/_454b261869.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:05:27.557Z","updatedAt":"2022-05-12T11:05:27.557Z"}}},"localizations":{"data":[{"id":32,"attributes":{"name":"当当","website":null,"createdAt":"2022-05-12T11:05:45.489Z","updatedAt":"2022-05-12T11:05:45.489Z","locale":"en"}}]}}},{"id":35,"attributes":{"name":"丁香园","website":null,"createdAt":"2022-05-12T11:06:42.076Z","updatedAt":"2022-05-12T11:07:02.931Z","locale":"zh","logo":{"data":{"id":204,"attributes":{"name":"丁香园.png","alternativeText":"丁香园.png","caption":"丁香园.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_丁香园.png","hash":"thumbnail__bd4781b4c7","ext":".png","mime":"image/png","width":245,"height":73,"size":6.46,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__bd4781b4c7.png"}},"hash":"_bd4781b4c7","ext":".png","mime":"image/png","size":9.79,"url":"https://official-media.sphere-ex.com/_bd4781b4c7.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:06:35.002Z","updatedAt":"2022-05-12T11:06:35.002Z"}}},"localizations":{"data":[{"id":36,"attributes":{"name":"丁香园","website":null,"createdAt":"2022-05-12T11:07:02.905Z","updatedAt":"2022-05-12T11:07:02.905Z","locale":"en"}}]}}},{"id":37,"attributes":{"name":"得物","website":null,"createdAt":"2022-05-12T11:08:13.051Z","updatedAt":"2022-05-12T11:08:25.837Z","locale":"zh","logo":{"data":{"id":203,"attributes":{"name":"得物.png","alternativeText":"得物.png","caption":"得物.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_得物.png","hash":"thumbnail__3625fb8574","ext":".png","mime":"image/png","width":245,"height":73,"size":3.52,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__3625fb8574.png"}},"hash":"_3625fb8574","ext":".png","mime":"image/png","size":3.77,"url":"https://official-media.sphere-ex.com/_3625fb8574.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:06:06.032Z","updatedAt":"2022-05-12T11:06:06.032Z"}}},"localizations":{"data":[{"id":38,"attributes":{"name":"得物","website":null,"createdAt":"2022-05-12T11:08:25.812Z","updatedAt":"2022-05-12T11:08:25.812Z","locale":"en"}}]}}},{"id":40,"attributes":{"name":"叮当快药","website":null,"createdAt":"2022-05-12T11:09:38.353Z","updatedAt":"2022-05-12T11:09:38.353Z","locale":"zh","logo":{"data":{"id":206,"attributes":{"name":"叮当快药.png","alternativeText":"叮当快药.png","caption":"叮当快药.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_叮当快药.png","hash":"thumbnail__0f02ae8949","ext":".png","mime":"image/png","width":245,"height":73,"size":7.88,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__0f02ae8949.png"}},"hash":"_0f02ae8949","ext":".png","mime":"image/png","size":14.81,"url":"https://official-media.sphere-ex.com/_0f02ae8949.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:09:20.787Z","updatedAt":"2022-05-12T11:09:20.787Z"}}},"localizations":{"data":[{"id":39,"attributes":{"name":"叮当快药","website":null,"createdAt":"2022-05-12T11:09:30.322Z","updatedAt":"2022-05-12T11:09:38.381Z","locale":"en"}}]}}},{"id":41,"attributes":{"name":"斗鱼","website":null,"createdAt":"2022-05-12T11:10:11.928Z","updatedAt":"2022-05-12T11:10:22.618Z","locale":"zh","logo":{"data":{"id":207,"attributes":{"name":"斗鱼.png","alternativeText":"斗鱼.png","caption":"斗鱼.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_斗鱼.png","hash":"thumbnail__50476865dc","ext":".png","mime":"image/png","width":245,"height":73,"size":11.3,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__50476865dc.png"}},"hash":"_50476865dc","ext":".png","mime":"image/png","size":22.41,"url":"https://official-media.sphere-ex.com/_50476865dc.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:10:03.411Z","updatedAt":"2022-05-12T11:10:03.411Z"}}},"localizations":{"data":[{"id":42,"attributes":{"name":"斗鱼","website":null,"createdAt":"2022-05-12T11:10:22.593Z","updatedAt":"2022-05-12T11:10:22.593Z","locale":"en"}}]}}},{"id":43,"attributes":{"name":"公牛","website":null,"createdAt":"2022-05-12T11:10:47.512Z","updatedAt":"2022-05-12T11:10:57.005Z","locale":"zh","logo":{"data":{"id":208,"attributes":{"name":"公牛.png","alternativeText":"公牛.png","caption":"公牛.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_公牛.png","hash":"thumbnail__f5c6be9842","ext":".png","mime":"image/png","width":245,"height":73,"size":6.26,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__f5c6be9842.png"}},"hash":"_f5c6be9842","ext":".png","mime":"image/png","size":10.15,"url":"https://official-media.sphere-ex.com/_f5c6be9842.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:10:36.578Z","updatedAt":"2022-05-12T11:10:36.578Z"}}},"localizations":{"data":[{"id":44,"attributes":{"name":"公牛","website":null,"createdAt":"2022-05-12T11:10:56.977Z","updatedAt":"2022-05-12T11:10:56.977Z","locale":"en"}}]}}},{"id":45,"attributes":{"name":"海典","website":null,"createdAt":"2022-05-12T11:11:32.952Z","updatedAt":"2022-05-12T11:14:54.887Z","locale":"zh","logo":{"data":{"id":209,"attributes":{"name":"海典.png","alternativeText":"海典.png","caption":"海典.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_海典.png","hash":"thumbnail__ce8f7e0cc1","ext":".png","mime":"image/png","width":245,"height":73,"size":8.02,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__ce8f7e0cc1.png"}},"hash":"_ce8f7e0cc1","ext":".png","mime":"image/png","size":10.81,"url":"https://official-media.sphere-ex.com/_ce8f7e0cc1.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:11:18.684Z","updatedAt":"2022-05-12T11:11:18.684Z"}}},"localizations":{"data":[{"id":46,"attributes":{"name":"海典","website":null,"createdAt":"2022-05-12T11:11:45.328Z","updatedAt":"2022-05-12T11:14:58.768Z","locale":"en"}}]}}},{"id":47,"attributes":{"name":"广联达","website":null,"createdAt":"2022-05-12T11:11:59.223Z","updatedAt":"2022-05-12T11:12:09.270Z","locale":"zh","logo":{"data":{"id":210,"attributes":{"name":"广联达.png","alternativeText":"广联达.png","caption":"广联达.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_广联达.png","hash":"thumbnail__5b3f63fe0c","ext":".png","mime":"image/png","width":245,"height":73,"size":11.9,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__5b3f63fe0c.png"}},"hash":"_5b3f63fe0c","ext":".png","mime":"image/png","size":24.1,"url":"https://official-media.sphere-ex.com/_5b3f63fe0c.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:11:18.691Z","updatedAt":"2022-05-12T11:11:18.691Z"}}},"localizations":{"data":[{"id":48,"attributes":{"name":"广联达","website":null,"createdAt":"2022-05-12T11:12:09.243Z","updatedAt":"2022-05-12T11:12:09.243Z","locale":"en"}}]}}},{"id":49,"attributes":{"name":"海尔消费金融","website":null,"createdAt":"2022-05-12T11:12:29.224Z","updatedAt":"2022-05-12T11:12:39.803Z","locale":"zh","logo":{"data":{"id":211,"attributes":{"name":"海尔.png","alternativeText":"海尔.png","caption":"海尔.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_海尔.png","hash":"thumbnail__21ad0931e1","ext":".png","mime":"image/png","width":245,"height":73,"size":8.32,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__21ad0931e1.png"}},"hash":"_21ad0931e1","ext":".png","mime":"image/png","size":13.4,"url":"https://official-media.sphere-ex.com/_21ad0931e1.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:11:18.777Z","updatedAt":"2022-05-12T11:11:18.777Z"}}},"localizations":{"data":[{"id":50,"attributes":{"name":"海尔消费金融","website":null,"createdAt":"2022-05-12T11:12:39.776Z","updatedAt":"2022-05-12T11:12:39.776Z","locale":"en"}}]}}},{"id":51,"attributes":{"name":"瓜子二手车","website":null,"createdAt":"2022-05-12T11:13:02.908Z","updatedAt":"2022-05-12T11:13:19.943Z","locale":"zh","logo":{"data":{"id":212,"attributes":{"name":"瓜子.png","alternativeText":"瓜子.png","caption":"瓜子.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_瓜子.png","hash":"thumbnail__ac83d8efd6","ext":".png","mime":"image/png","width":245,"height":73,"size":11.82,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__ac83d8efd6.png"}},"hash":"_ac83d8efd6","ext":".png","mime":"image/png","size":22.1,"url":"https://official-media.sphere-ex.com/_ac83d8efd6.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:11:18.862Z","updatedAt":"2022-05-12T11:11:18.862Z"}}},"localizations":{"data":[{"id":52,"attributes":{"name":"瓜子二手车","website":null,"createdAt":"2022-05-12T11:13:19.919Z","updatedAt":"2022-05-12T11:13:19.919Z","locale":"en"}}]}}},{"id":53,"attributes":{"name":"怪兽充电","website":null,"createdAt":"2022-05-12T11:13:38.051Z","updatedAt":"2022-05-12T11:13:46.687Z","locale":"zh","logo":{"data":{"id":213,"attributes":{"name":"怪兽充电.png","alternativeText":"怪兽充电.png","caption":"怪兽充电.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_怪兽充电.png","hash":"thumbnail__a71d67b0e7","ext":".png","mime":"image/png","width":245,"height":73,"size":8.74,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__a71d67b0e7.png"}},"hash":"_a71d67b0e7","ext":".png","mime":"image/png","size":15.71,"url":"https://official-media.sphere-ex.com/_a71d67b0e7.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:11:18.863Z","updatedAt":"2022-05-12T11:11:18.863Z"}}},"localizations":{"data":[{"id":54,"attributes":{"name":"怪兽充电","website":null,"createdAt":"2022-05-12T11:13:46.662Z","updatedAt":"2022-05-12T11:13:46.662Z","locale":"en"}}]}}},{"id":55,"attributes":{"name":"海康","website":null,"createdAt":"2022-05-12T11:14:20.838Z","updatedAt":"2022-05-12T11:14:35.835Z","locale":"zh","logo":{"data":{"id":214,"attributes":{"name":"海康威视.png","alternativeText":"海康威视.png","caption":"海康威视.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_海康威视.png","hash":"thumbnail__3fd9901f86","ext":".png","mime":"image/png","width":245,"height":73,"size":6.17,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__3fd9901f86.png"}},"hash":"_3fd9901f86","ext":".png","mime":"image/png","size":8.15,"url":"https://official-media.sphere-ex.com/_3fd9901f86.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:14:12.210Z","updatedAt":"2022-05-12T11:14:12.210Z"}}},"localizations":{"data":[{"id":56,"attributes":{"name":"海康","website":null,"createdAt":"2022-05-12T11:14:35.811Z","updatedAt":"2022-05-12T11:14:35.811Z","locale":"en"}}]}}},{"id":57,"attributes":{"name":"好未来","website":null,"createdAt":"2022-05-12T11:15:53.432Z","updatedAt":"2022-05-12T11:16:04.478Z","locale":"zh","logo":{"data":{"id":215,"attributes":{"name":"好未来.png","alternativeText":"好未来.png","caption":"好未来.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_好未来.png","hash":"thumbnail__d93086d051","ext":".png","mime":"image/png","width":245,"height":73,"size":10.17,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__d93086d051.png"}},"hash":"_d93086d051","ext":".png","mime":"image/png","size":20.34,"url":"https://official-media.sphere-ex.com/_d93086d051.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:15:46.920Z","updatedAt":"2022-05-12T11:15:46.920Z"}}},"localizations":{"data":[{"id":58,"attributes":{"name":"好未来","website":null,"createdAt":"2022-05-12T11:16:04.452Z","updatedAt":"2022-05-12T11:16:04.452Z","locale":"en"}}]}}},{"id":59,"attributes":{"name":"欢聚时代","website":null,"createdAt":"2022-05-12T11:16:21.597Z","updatedAt":"2022-05-12T11:16:27.246Z","locale":"zh","logo":{"data":{"id":216,"attributes":{"name":"欢聚时代.png","alternativeText":"欢聚时代.png","caption":"欢聚时代.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_欢聚时代.png","hash":"thumbnail__b286bdb96d","ext":".png","mime":"image/png","width":245,"height":73,"size":12.22,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__b286bdb96d.png"}},"hash":"_b286bdb96d","ext":".png","mime":"image/png","size":22.36,"url":"https://official-media.sphere-ex.com/_b286bdb96d.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:16:14.879Z","updatedAt":"2022-05-12T11:16:14.879Z"}}},"localizations":{"data":[{"id":60,"attributes":{"name":"欢聚时代","website":null,"createdAt":"2022-05-12T11:16:27.223Z","updatedAt":"2022-05-12T11:16:27.223Z","locale":"en"}}]}}},{"id":61,"attributes":{"name":"环球网校","website":null,"createdAt":"2022-05-12T11:16:58.746Z","updatedAt":"2022-05-12T11:17:07.952Z","locale":"zh","logo":{"data":{"id":217,"attributes":{"name":"环球网校.png","alternativeText":"环球网校.png","caption":"环球网校.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_环球网校.png","hash":"thumbnail__57f58a8c7b","ext":".png","mime":"image/png","width":245,"height":73,"size":8.77,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__57f58a8c7b.png"}},"hash":"_57f58a8c7b","ext":".png","mime":"image/png","size":15.82,"url":"https://official-media.sphere-ex.com/_57f58a8c7b.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:16:46.576Z","updatedAt":"2022-05-12T11:16:46.576Z"}}},"localizations":{"data":[{"id":62,"attributes":{"name":"环球网校","website":null,"createdAt":"2022-05-12T11:17:07.929Z","updatedAt":"2022-05-12T11:17:07.929Z","locale":"en"}}]}}},{"id":63,"attributes":{"name":"货车帮","website":null,"createdAt":"2022-05-12T11:17:31.439Z","updatedAt":"2022-05-12T11:17:37.868Z","locale":"zh","logo":{"data":{"id":218,"attributes":{"name":"货车帮.png","alternativeText":"货车帮.png","caption":"货车帮.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_货车帮.png","hash":"thumbnail__d7cbf73239","ext":".png","mime":"image/png","width":245,"height":73,"size":11.36,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__d7cbf73239.png"}},"hash":"_d7cbf73239","ext":".png","mime":"image/png","size":17.92,"url":"https://official-media.sphere-ex.com/_d7cbf73239.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:17:23.196Z","updatedAt":"2022-05-12T11:17:23.196Z"}}},"localizations":{"data":[{"id":64,"attributes":{"name":"货车帮","website":null,"createdAt":"2022-05-12T11:17:37.845Z","updatedAt":"2022-05-12T11:17:37.845Z","locale":"en"}}]}}},{"id":65,"attributes":{"name":"家视天下","website":null,"createdAt":"2022-05-12T11:18:06.319Z","updatedAt":"2022-05-12T11:18:16.588Z","locale":"zh","logo":{"data":{"id":219,"attributes":{"name":"家视天下.png","alternativeText":"家视天下.png","caption":"家视天下.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_家视天下.png","hash":"thumbnail__69bf270b74","ext":".png","mime":"image/png","width":245,"height":73,"size":5.13,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__69bf270b74.png"}},"hash":"_69bf270b74","ext":".png","mime":"image/png","size":7.15,"url":"https://official-media.sphere-ex.com/_69bf270b74.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:17:54.407Z","updatedAt":"2022-05-12T11:17:54.407Z"}}},"localizations":{"data":[{"id":66,"attributes":{"name":"家视天下","website":null,"createdAt":"2022-05-12T11:18:16.566Z","updatedAt":"2022-05-12T11:18:16.566Z","locale":"en"}}]}}},{"id":67,"attributes":{"name":"建信金科","website":null,"createdAt":"2022-05-12T11:18:42.863Z","updatedAt":"2022-05-12T11:18:49.256Z","locale":"zh","logo":{"data":{"id":220,"attributes":{"name":"建信金科.png","alternativeText":"建信金科.png","caption":"建信金科.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_建信金科.png","hash":"thumbnail__6b2c67ba48","ext":".png","mime":"image/png","width":245,"height":73,"size":9.45,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__6b2c67ba48.png"}},"hash":"_6b2c67ba48","ext":".png","mime":"image/png","size":15.25,"url":"https://official-media.sphere-ex.com/_6b2c67ba48.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:18:33.807Z","updatedAt":"2022-05-12T11:18:33.807Z"}}},"localizations":{"data":[{"id":68,"attributes":{"name":"建信金科","website":null,"createdAt":"2022-05-12T11:18:49.234Z","updatedAt":"2022-05-12T11:18:49.234Z","locale":"en"}}]}}},{"id":69,"attributes":{"name":"京东科技","website":null,"createdAt":"2022-05-12T11:19:09.045Z","updatedAt":"2022-05-12T11:19:17.099Z","locale":"zh","logo":{"data":{"id":221,"attributes":{"name":"京东科技.png","alternativeText":"京东科技.png","caption":"京东科技.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_京东科技.png","hash":"thumbnail__36be828c9c","ext":".png","mime":"image/png","width":245,"height":73,"size":7.44,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__36be828c9c.png"}},"hash":"_36be828c9c","ext":".png","mime":"image/png","size":11.23,"url":"https://official-media.sphere-ex.com/_36be828c9c.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:19:02.366Z","updatedAt":"2022-05-12T11:19:02.366Z"}}},"localizations":{"data":[{"id":70,"attributes":{"name":"京东科技","website":null,"createdAt":"2022-05-12T11:19:17.074Z","updatedAt":"2022-05-12T11:19:17.074Z","locale":"en"}}]}}},{"id":71,"attributes":{"name":"京东物流","website":null,"createdAt":"2022-05-12T11:19:36.772Z","updatedAt":"2022-05-12T11:19:48.204Z","locale":"zh","logo":{"data":{"id":222,"attributes":{"name":"京东物流.png","alternativeText":"京东物流.png","caption":"京东物流.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_京东物流.png","hash":"thumbnail__20b318c172","ext":".png","mime":"image/png","width":245,"height":73,"size":6.21,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__20b318c172.png"}},"hash":"_20b318c172","ext":".png","mime":"image/png","size":5.83,"url":"https://official-media.sphere-ex.com/_20b318c172.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:19:28.897Z","updatedAt":"2022-05-12T11:19:28.897Z"}}},"localizations":{"data":[{"id":72,"attributes":{"name":"京东物流","website":null,"createdAt":"2022-05-12T11:19:48.106Z","updatedAt":"2022-05-12T11:19:48.106Z","locale":"en"}}]}}},{"id":73,"attributes":{"name":"玖富集团","website":null,"createdAt":"2022-05-12T11:20:14.253Z","updatedAt":"2022-05-12T11:20:21.264Z","locale":"zh","logo":{"data":{"id":223,"attributes":{"name":"玖富集团.png","alternativeText":"玖富集团.png","caption":"玖富集团.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_玖富集团.png","hash":"thumbnail__a9a3ec5822","ext":".png","mime":"image/png","width":245,"height":73,"size":7.26,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__a9a3ec5822.png"}},"hash":"_a9a3ec5822","ext":".png","mime":"image/png","size":11.86,"url":"https://official-media.sphere-ex.com/_a9a3ec5822.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:20:00.653Z","updatedAt":"2022-05-12T11:20:00.653Z"}}},"localizations":{"data":[{"id":74,"attributes":{"name":"玖富集团","website":null,"createdAt":"2022-05-12T11:20:21.242Z","updatedAt":"2022-05-12T11:20:21.242Z","locale":"en"}}]}}},{"id":75,"attributes":{"name":"科大讯飞","website":null,"createdAt":"2022-05-12T11:20:39.898Z","updatedAt":"2022-05-12T11:20:45.857Z","locale":"zh","logo":{"data":{"id":224,"attributes":{"name":"科大讯飞.png","alternativeText":"科大讯飞.png","caption":"科大讯飞.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_科大讯飞.png","hash":"thumbnail__840f032a53","ext":".png","mime":"image/png","width":245,"height":73,"size":8.73,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__840f032a53.png"}},"hash":"_840f032a53","ext":".png","mime":"image/png","size":12.8,"url":"https://official-media.sphere-ex.com/_840f032a53.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:20:32.329Z","updatedAt":"2022-05-12T11:20:32.329Z"}}},"localizations":{"data":[{"id":76,"attributes":{"name":"科大讯飞","website":null,"createdAt":"2022-05-12T11:20:45.826Z","updatedAt":"2022-05-12T11:20:45.826Z","locale":"en"}}]}}},{"id":77,"attributes":{"name":"客如云","website":null,"createdAt":"2022-05-12T11:21:06.372Z","updatedAt":"2022-05-12T11:21:12.174Z","locale":"zh","logo":{"data":{"id":225,"attributes":{"name":"客如云.png","alternativeText":"客如云.png","caption":"客如云.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_客如云.png","hash":"thumbnail__9539ee7440","ext":".png","mime":"image/png","width":245,"height":73,"size":5.78,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__9539ee7440.png"}},"hash":"_9539ee7440","ext":".png","mime":"image/png","size":4.81,"url":"https://official-media.sphere-ex.com/_9539ee7440.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:20:57.667Z","updatedAt":"2022-05-12T11:20:57.667Z"}}},"localizations":{"data":[{"id":78,"attributes":{"name":"客如云","website":null,"createdAt":"2022-05-12T11:21:12.112Z","updatedAt":"2022-05-12T11:21:12.112Z","locale":"en"}}]}}},{"id":79,"attributes":{"name":"马上消费","website":null,"createdAt":"2022-05-12T11:21:50.193Z","updatedAt":"2022-05-12T11:22:06.902Z","locale":"zh","logo":{"data":{"id":226,"attributes":{"name":"马上消费.png","alternativeText":"马上消费.png","caption":"马上消费.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_马上消费.png","hash":"thumbnail__96b249b440","ext":".png","mime":"image/png","width":245,"height":73,"size":7.5,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__96b249b440.png"}},"hash":"_96b249b440","ext":".png","mime":"image/png","size":11.76,"url":"https://official-media.sphere-ex.com/_96b249b440.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:21:41.147Z","updatedAt":"2022-05-12T11:21:41.147Z"}}},"localizations":{"data":[{"id":80,"attributes":{"name":"马上消费","website":null,"createdAt":"2022-05-12T11:22:06.877Z","updatedAt":"2022-05-12T11:22:06.877Z","locale":"en"}}]}}},{"id":81,"attributes":{"name":"联通云","website":null,"createdAt":"2022-05-12T11:22:25.376Z","updatedAt":"2022-05-12T11:22:36.391Z","locale":"zh","logo":{"data":{"id":227,"attributes":{"name":"联通云.png","alternativeText":"联通云.png","caption":"联通云.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_联通云.png","hash":"thumbnail__b834778d2b","ext":".png","mime":"image/png","width":245,"height":73,"size":8.03,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__b834778d2b.png"}},"hash":"_b834778d2b","ext":".png","mime":"image/png","size":14.26,"url":"https://official-media.sphere-ex.com/_b834778d2b.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:21:41.228Z","updatedAt":"2022-05-12T11:21:41.228Z"}}},"localizations":{"data":[{"id":82,"attributes":{"name":"联通云","website":null,"createdAt":"2022-05-12T11:22:36.369Z","updatedAt":"2022-05-12T11:22:36.369Z","locale":"en"}}]}}},{"id":83,"attributes":{"name":"马蜂窝","website":null,"createdAt":"2022-05-12T11:22:56.870Z","updatedAt":"2022-05-12T11:23:07.683Z","locale":"zh","logo":{"data":{"id":228,"attributes":{"name":"马蜂窝.png","alternativeText":"马蜂窝.png","caption":"马蜂窝.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_马蜂窝.png","hash":"thumbnail__3c4a989dc8","ext":".png","mime":"image/png","width":245,"height":73,"size":8.87,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__3c4a989dc8.png"}},"hash":"_3c4a989dc8","ext":".png","mime":"image/png","size":15.92,"url":"https://official-media.sphere-ex.com/_3c4a989dc8.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:21:41.300Z","updatedAt":"2022-05-12T11:21:41.300Z"}}},"localizations":{"data":[{"id":84,"attributes":{"name":"马蜂窝","website":null,"createdAt":"2022-05-12T11:23:07.661Z","updatedAt":"2022-05-12T11:23:07.661Z","locale":"en"}}]}}},{"id":85,"attributes":{"name":"快捷通","website":null,"createdAt":"2022-05-12T11:23:30.037Z","updatedAt":"2022-05-12T11:23:37.812Z","locale":"zh","logo":{"data":{"id":229,"attributes":{"name":"快捷通.png","alternativeText":"快捷通.png","caption":"快捷通.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_快捷通.png","hash":"thumbnail__7a0f85a547","ext":".png","mime":"image/png","width":245,"height":73,"size":7.73,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__7a0f85a547.png"}},"hash":"_7a0f85a547","ext":".png","mime":"image/png","size":11.36,"url":"https://official-media.sphere-ex.com/_7a0f85a547.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:21:41.388Z","updatedAt":"2022-05-12T11:21:41.388Z"}}},"localizations":{"data":[{"id":86,"attributes":{"name":"快捷通","website":null,"createdAt":"2022-05-12T11:23:37.789Z","updatedAt":"2022-05-12T11:23:37.789Z","locale":"en"}}]}}},{"id":87,"attributes":{"name":"理想","website":null,"createdAt":"2022-05-12T11:23:56.121Z","updatedAt":"2022-05-12T11:24:03.523Z","locale":"zh","logo":{"data":{"id":230,"attributes":{"name":"理想.png","alternativeText":"理想.png","caption":"理想.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_理想.png","hash":"thumbnail__e076874d2d","ext":".png","mime":"image/png","width":245,"height":73,"size":6.59,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__e076874d2d.png"}},"hash":"_e076874d2d","ext":".png","mime":"image/png","size":11.85,"url":"https://official-media.sphere-ex.com/_e076874d2d.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:21:41.511Z","updatedAt":"2022-05-12T11:21:41.511Z"}}},"localizations":{"data":[{"id":88,"attributes":{"name":"理想","website":null,"createdAt":"2022-05-12T11:24:03.499Z","updatedAt":"2022-05-12T11:24:03.499Z","locale":"en"}}]}}},{"id":89,"attributes":{"name":"每日优鲜","website":null,"createdAt":"2022-05-12T11:30:58.535Z","updatedAt":"2022-05-12T11:31:06.422Z","locale":"zh","logo":{"data":{"id":231,"attributes":{"name":"每日优鲜.png","alternativeText":"每日优鲜.png","caption":"每日优鲜.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_每日优鲜.png","hash":"thumbnail__8049aa0985","ext":".png","mime":"image/png","width":245,"height":73,"size":8.03,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__8049aa0985.png"}},"hash":"_8049aa0985","ext":".png","mime":"image/png","size":12.69,"url":"https://official-media.sphere-ex.com/_8049aa0985.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:30:49.874Z","updatedAt":"2022-05-12T11:30:49.874Z"}}},"localizations":{"data":[{"id":90,"attributes":{"name":"每日优鲜","website":null,"createdAt":"2022-05-12T11:31:06.401Z","updatedAt":"2022-05-12T11:31:06.401Z","locale":"en"}}]}}},{"id":92,"attributes":{"name":"中国民生银行","website":null,"createdAt":"2022-05-13T08:53:45.974Z","updatedAt":"2022-05-13T08:53:45.974Z","locale":"zh","logo":{"data":{"id":232,"attributes":{"name":"民生银行.png","alternativeText":"民生银行.png","caption":"民生银行.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_民生银行.png","hash":"thumbnail__6fd163ced8","ext":".png","mime":"image/png","width":245,"height":73,"size":12.39,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__6fd163ced8.png"}},"hash":"_6fd163ced8","ext":".png","mime":"image/png","size":23.87,"url":"https://official-media.sphere-ex.com/_6fd163ced8.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:53:14.337Z","updatedAt":"2022-05-13T08:53:14.337Z"}}},"localizations":{"data":[{"id":91,"attributes":{"name":"中国民生银行","website":null,"createdAt":"2022-05-13T08:53:26.029Z","updatedAt":"2022-05-13T08:53:46.005Z","locale":"en"}}]}}},{"id":93,"attributes":{"name":"搜狐新闻","website":null,"createdAt":"2022-05-13T08:54:19.289Z","updatedAt":"2022-05-13T08:54:28.945Z","locale":"zh","logo":{"data":{"id":234,"attributes":{"name":"搜狐新闻.png","alternativeText":"搜狐新闻.png","caption":"搜狐新闻.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_搜狐新闻.png","hash":"thumbnail__f06beae44d","ext":".png","mime":"image/png","width":245,"height":73,"size":8.08,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__f06beae44d.png"}},"hash":"_f06beae44d","ext":".png","mime":"image/png","size":14.43,"url":"https://official-media.sphere-ex.com/_f06beae44d.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:54:12.179Z","updatedAt":"2022-05-13T08:54:12.179Z"}}},"localizations":{"data":[{"id":94,"attributes":{"name":"搜狐新闻","website":null,"createdAt":"2022-05-13T08:54:28.895Z","updatedAt":"2022-05-13T08:54:28.895Z","locale":"en"}}]}}},{"id":95,"attributes":{"name":"汽车之家","website":null,"createdAt":"2022-05-13T08:54:40.671Z","updatedAt":"2022-05-13T08:54:51.370Z","locale":"zh","logo":{"data":{"id":233,"attributes":{"name":"汽车之家.png","alternativeText":"汽车之家.png","caption":"汽车之家.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_汽车之家.png","hash":"thumbnail__52e1c6df2d","ext":".png","mime":"image/png","width":245,"height":73,"size":11.84,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__52e1c6df2d.png"}},"hash":"_52e1c6df2d","ext":".png","mime":"image/png","size":20.7,"url":"https://official-media.sphere-ex.com/_52e1c6df2d.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:54:12.154Z","updatedAt":"2022-05-13T08:54:12.154Z"}}},"localizations":{"data":[{"id":96,"attributes":{"name":"汽车之家","website":null,"createdAt":"2022-05-13T08:54:51.347Z","updatedAt":"2022-05-13T08:54:51.347Z","locale":"en"}}]}}},{"id":97,"attributes":{"name":"闪送","website":null,"createdAt":"2022-05-13T08:55:04.149Z","updatedAt":"2022-05-13T08:55:10.674Z","locale":"zh","logo":{"data":{"id":235,"attributes":{"name":"闪送.png","alternativeText":"闪送.png","caption":"闪送.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_闪送.png","hash":"thumbnail__43705d7b17","ext":".png","mime":"image/png","width":245,"height":73,"size":15.58,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__43705d7b17.png"}},"hash":"_43705d7b17","ext":".png","mime":"image/png","size":31.84,"url":"https://official-media.sphere-ex.com/_43705d7b17.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:54:12.223Z","updatedAt":"2022-05-13T08:54:12.223Z"}}},"localizations":{"data":[{"id":98,"attributes":{"name":"闪送","website":null,"createdAt":"2022-05-13T08:55:10.617Z","updatedAt":"2022-05-13T08:55:10.617Z","locale":"en"}}]}}},{"id":99,"attributes":{"name":"三只松鼠","website":null,"createdAt":"2022-05-13T08:55:21.083Z","updatedAt":"2022-05-13T08:55:28.282Z","locale":"zh","logo":{"data":{"id":236,"attributes":{"name":"三只松鼠.png","alternativeText":"三只松鼠.png","caption":"三只松鼠.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_三只松鼠.png","hash":"thumbnail__8c33dfeb6b","ext":".png","mime":"image/png","width":245,"height":73,"size":11.61,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__8c33dfeb6b.png"}},"hash":"_8c33dfeb6b","ext":".png","mime":"image/png","size":22.67,"url":"https://official-media.sphere-ex.com/_8c33dfeb6b.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:54:12.230Z","updatedAt":"2022-05-13T08:54:12.230Z"}}},"localizations":{"data":[{"id":100,"attributes":{"name":"三只松鼠","website":null,"createdAt":"2022-05-13T08:55:28.254Z","updatedAt":"2022-05-13T08:55:28.254Z","locale":"en"}}]}}},{"id":101,"attributes":{"name":"首汽约车","website":null,"createdAt":"2022-05-13T08:55:44.940Z","updatedAt":"2022-05-13T08:55:52.567Z","locale":"zh","logo":{"data":{"id":237,"attributes":{"name":"首汽约车.png","alternativeText":"首汽约车.png","caption":"首汽约车.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_首汽约车.png","hash":"thumbnail__ca02385dfc","ext":".png","mime":"image/png","width":245,"height":73,"size":10.92,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__ca02385dfc.png"}},"hash":"_ca02385dfc","ext":".png","mime":"image/png","size":21.38,"url":"https://official-media.sphere-ex.com/_ca02385dfc.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:54:12.374Z","updatedAt":"2022-05-13T08:54:12.374Z"}}},"localizations":{"data":[{"id":102,"attributes":{"name":"首汽约车","website":null,"createdAt":"2022-05-13T08:55:52.546Z","updatedAt":"2022-05-13T08:55:52.546Z","locale":"en"}}]}}},{"id":103,"attributes":{"name":"腾讯音乐人","website":null,"createdAt":"2022-05-13T08:56:30.799Z","updatedAt":"2022-05-13T08:56:37.803Z","locale":"zh","logo":{"data":{"id":238,"attributes":{"name":"腾讯音乐人.png","alternativeText":"腾讯音乐人.png","caption":"腾讯音乐人.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_腾讯音乐人.png","hash":"thumbnail__691253dfe6","ext":".png","mime":"image/png","width":245,"height":73,"size":7.17,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__691253dfe6.png"}},"hash":"_691253dfe6","ext":".png","mime":"image/png","size":10.04,"url":"https://official-media.sphere-ex.com/_691253dfe6.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:56:21.308Z","updatedAt":"2022-05-13T08:56:21.308Z"}}},"localizations":{"data":[{"id":104,"attributes":{"name":"腾讯音乐人","website":null,"createdAt":"2022-05-13T08:56:37.773Z","updatedAt":"2022-05-13T08:56:37.773Z","locale":"en"}}]}}},{"id":105,"attributes":{"name":"豌豆公主","website":null,"createdAt":"2022-05-13T08:56:49.892Z","updatedAt":"2022-05-13T08:56:56.478Z","locale":"zh","logo":{"data":{"id":239,"attributes":{"name":"豌豆公主.png","alternativeText":"豌豆公主.png","caption":"豌豆公主.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_豌豆公主.png","hash":"thumbnail__87d194643d","ext":".png","mime":"image/png","width":245,"height":73,"size":6.47,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__87d194643d.png"}},"hash":"_87d194643d","ext":".png","mime":"image/png","size":10.93,"url":"https://official-media.sphere-ex.com/_87d194643d.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:56:21.441Z","updatedAt":"2022-05-13T08:56:21.441Z"}}},"localizations":{"data":[{"id":106,"attributes":{"name":"豌豆公主","website":null,"createdAt":"2022-05-13T08:56:56.457Z","updatedAt":"2022-05-13T08:56:56.457Z","locale":"en"}}]}}},{"id":107,"attributes":{"name":"天涯社区","website":null,"createdAt":"2022-05-13T08:57:19.133Z","updatedAt":"2022-05-13T11:14:57.190Z","locale":"zh","logo":{"data":{"id":240,"attributes":{"name":"天涯社区.png","alternativeText":"天涯社区.png","caption":"天涯社区.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_天涯社区.png","hash":"thumbnail__34fdcd8ea3","ext":".png","mime":"image/png","width":245,"height":73,"size":17.91,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__34fdcd8ea3.png"}},"hash":"_34fdcd8ea3","ext":".png","mime":"image/png","size":36.45,"url":"https://official-media.sphere-ex.com/_34fdcd8ea3.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:56:21.445Z","updatedAt":"2022-05-13T08:56:21.445Z"}}},"localizations":{"data":[{"id":124,"attributes":{"name":"天涯社区","website":null,"createdAt":"2022-05-13T11:14:57.164Z","updatedAt":"2022-05-13T11:14:57.164Z","locale":"en"}}]}}},{"id":108,"attributes":{"name":"特微智能","website":null,"createdAt":"2022-05-13T11:06:01.596Z","updatedAt":"2022-05-13T11:06:26.584Z","locale":"zh","logo":{"data":{"id":241,"attributes":{"name":"特微智能.png","alternativeText":"特微智能.png","caption":"特微智能.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_特微智能.png","hash":"thumbnail__a3819c29c7","ext":".png","mime":"image/png","width":245,"height":73,"size":10.14,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__a3819c29c7.png"}},"hash":"_a3819c29c7","ext":".png","mime":"image/png","size":16.2,"url":"https://official-media.sphere-ex.com/_a3819c29c7.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:56:21.461Z","updatedAt":"2022-05-13T08:56:21.461Z"}}},"localizations":{"data":[{"id":109,"attributes":{"name":"特微智能","website":null,"createdAt":"2022-05-13T11:06:26.555Z","updatedAt":"2022-05-13T11:06:26.555Z","locale":"en"}}]}}},{"id":110,"attributes":{"name":"甜橙金融","website":null,"createdAt":"2022-05-13T11:06:40.170Z","updatedAt":"2022-05-13T11:06:50.639Z","locale":"zh","logo":{"data":{"id":242,"attributes":{"name":"甜橙金融.png","alternativeText":"甜橙金融.png","caption":"甜橙金融.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_甜橙金融.png","hash":"thumbnail__448c66753b","ext":".png","mime":"image/png","width":245,"height":73,"size":14.46,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__448c66753b.png"}},"hash":"_448c66753b","ext":".png","mime":"image/png","size":27.59,"url":"https://official-media.sphere-ex.com/_448c66753b.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:56:21.476Z","updatedAt":"2022-05-13T08:56:21.476Z"}}},"localizations":{"data":[{"id":111,"attributes":{"name":"甜橙金融","website":null,"createdAt":"2022-05-13T11:06:50.620Z","updatedAt":"2022-05-13T11:06:50.620Z","locale":"en"}}]}}},{"id":112,"attributes":{"name":"网易游戏","website":null,"createdAt":"2022-05-13T11:07:17.490Z","updatedAt":"2022-05-13T11:07:27.108Z","locale":"zh","logo":{"data":{"id":243,"attributes":{"name":"网易游戏.png","alternativeText":"网易游戏.png","caption":"网易游戏.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_网易游戏.png","hash":"thumbnail__6c9d91cfde","ext":".png","mime":"image/png","width":245,"height":73,"size":14.59,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__6c9d91cfde.png"}},"hash":"_6c9d91cfde","ext":".png","mime":"image/png","size":30.86,"url":"https://official-media.sphere-ex.com/_6c9d91cfde.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:07:10.149Z","updatedAt":"2022-05-13T11:07:10.149Z"}}},"localizations":{"data":[{"id":113,"attributes":{"name":"网易游戏","website":null,"createdAt":"2022-05-13T11:07:27.085Z","updatedAt":"2022-05-13T11:07:27.085Z","locale":"en"}}]}}},{"id":114,"attributes":{"name":"唯品会","website":null,"createdAt":"2022-05-13T11:07:38.182Z","updatedAt":"2022-05-13T11:07:47.075Z","locale":"zh","logo":{"data":{"id":244,"attributes":{"name":"唯品会.png","alternativeText":"唯品会.png","caption":"唯品会.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_唯品会.png","hash":"thumbnail__73c4ab0fa5","ext":".png","mime":"image/png","width":245,"height":73,"size":7.59,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__73c4ab0fa5.png"}},"hash":"_73c4ab0fa5","ext":".png","mime":"image/png","size":9.44,"url":"https://official-media.sphere-ex.com/_73c4ab0fa5.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:07:10.159Z","updatedAt":"2022-05-13T11:07:10.159Z"}}},"localizations":{"data":[{"id":115,"attributes":{"name":"唯品会","website":null,"createdAt":"2022-05-13T11:07:47.052Z","updatedAt":"2022-05-13T11:07:47.052Z","locale":"en"}}]}}},{"id":116,"attributes":{"name":"携程","website":null,"createdAt":"2022-05-13T11:10:23.100Z","updatedAt":"2022-05-13T11:10:33.504Z","locale":"zh","logo":{"data":{"id":245,"attributes":{"name":"携程.png","alternativeText":"携程.png","caption":"携程.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_携程.png","hash":"thumbnail__b86b92e821","ext":".png","mime":"image/png","width":245,"height":73,"size":10.25,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__b86b92e821.png"}},"hash":"_b86b92e821","ext":".png","mime":"image/png","size":17.88,"url":"https://official-media.sphere-ex.com/_b86b92e821.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:10:15.339Z","updatedAt":"2022-05-13T11:10:15.339Z"}}},"localizations":{"data":[{"id":117,"attributes":{"name":"携程","website":null,"createdAt":"2022-05-13T11:10:33.475Z","updatedAt":"2022-05-13T11:10:33.475Z","locale":"en"}}]}}},{"id":118,"attributes":{"name":"易车","website":null,"createdAt":"2022-05-13T11:13:25.517Z","updatedAt":"2022-05-13T11:13:35.381Z","locale":"zh","logo":{"data":{"id":246,"attributes":{"name":"易车.png","alternativeText":"易车.png","caption":"易车.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_易车.png","hash":"thumbnail__e0cabfed8a","ext":".png","mime":"image/png","width":245,"height":73,"size":9.82,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__e0cabfed8a.png"}},"hash":"_e0cabfed8a","ext":".png","mime":"image/png","size":19.88,"url":"https://official-media.sphere-ex.com/_e0cabfed8a.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:13:19.094Z","updatedAt":"2022-05-13T11:13:19.094Z"}}},"localizations":{"data":[{"id":119,"attributes":{"name":"易车","website":null,"createdAt":"2022-05-13T11:13:35.354Z","updatedAt":"2022-05-13T11:13:35.354Z","locale":"en"}}]}}},{"id":120,"attributes":{"name":"云学堂","website":null,"createdAt":"2022-05-13T11:13:46.465Z","updatedAt":"2022-05-13T11:13:56.133Z","locale":"zh","logo":{"data":{"id":247,"attributes":{"name":"云学堂.png","alternativeText":"云学堂.png","caption":"云学堂.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_云学堂.png","hash":"thumbnail__38d46e9c1f","ext":".png","mime":"image/png","width":245,"height":73,"size":9.16,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__38d46e9c1f.png"}},"hash":"_38d46e9c1f","ext":".png","mime":"image/png","size":13.93,"url":"https://official-media.sphere-ex.com/_38d46e9c1f.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:13:19.212Z","updatedAt":"2022-05-13T11:13:19.212Z"}}},"localizations":{"data":[{"id":121,"attributes":{"name":"云学堂","website":null,"createdAt":"2022-05-13T11:13:56.105Z","updatedAt":"2022-05-13T11:13:56.105Z","locale":"en"}}]}}},{"id":122,"attributes":{"name":"韵达","website":null,"createdAt":"2022-05-13T11:14:22.755Z","updatedAt":"2022-05-13T11:14:30.758Z","locale":"zh","logo":{"data":{"id":248,"attributes":{"name":"韵达.png","alternativeText":"韵达.png","caption":"韵达.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_韵达.png","hash":"thumbnail__d887641689","ext":".png","mime":"image/png","width":245,"height":73,"size":5.93,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__d887641689.png"}},"hash":"_d887641689","ext":".png","mime":"image/png","size":5.19,"url":"https://official-media.sphere-ex.com/_d887641689.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:13:19.280Z","updatedAt":"2022-05-13T11:13:19.280Z"}}},"localizations":{"data":[{"id":123,"attributes":{"name":"韵达","website":null,"createdAt":"2022-05-13T11:14:30.735Z","updatedAt":"2022-05-13T11:14:30.735Z","locale":"en"}}]}}},{"id":125,"attributes":{"name":"元气森林","website":null,"createdAt":"2022-05-13T11:15:18.862Z","updatedAt":"2022-05-13T11:15:29.267Z","locale":"zh","logo":{"data":{"id":249,"attributes":{"name":"元气森林.png","alternativeText":"元气森林.png","caption":"元气森林.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_元气森林.png","hash":"thumbnail__13df5999cb","ext":".png","mime":"image/png","width":245,"height":73,"size":5.21,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__13df5999cb.png"}},"hash":"_13df5999cb","ext":".png","mime":"image/png","size":9.91,"url":"https://official-media.sphere-ex.com/_13df5999cb.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:13:19.354Z","updatedAt":"2022-05-13T11:13:19.354Z"}}},"localizations":{"data":[{"id":126,"attributes":{"name":"元气森林","website":null,"createdAt":"2022-05-13T11:15:29.243Z","updatedAt":"2022-05-13T11:15:29.243Z","locale":"en"}}]}}},{"id":127,"attributes":{"name":"中通快递","website":null,"createdAt":"2022-05-13T11:16:17.997Z","updatedAt":"2022-05-13T11:16:29.091Z","locale":"zh","logo":{"data":{"id":250,"attributes":{"name":"中通快递.png","alternativeText":"中通快递.png","caption":"中通快递.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_中通快递.png","hash":"thumbnail__dfc99996ce","ext":".png","mime":"image/png","width":245,"height":73,"size":8.56,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__dfc99996ce.png"}},"hash":"_dfc99996ce","ext":".png","mime":"image/png","size":15.26,"url":"https://official-media.sphere-ex.com/_dfc99996ce.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:16:10.523Z","updatedAt":"2022-05-13T11:16:10.523Z"}}},"localizations":{"data":[{"id":128,"attributes":{"name":"中通快递","website":null,"createdAt":"2022-05-13T11:16:29.062Z","updatedAt":"2022-05-13T11:16:29.062Z","locale":"en"}}]}}},{"id":129,"attributes":{"name":"bilibili","website":null,"createdAt":"2022-05-13T11:16:55.526Z","updatedAt":"2022-05-13T11:17:02.244Z","locale":"zh","logo":{"data":{"id":251,"attributes":{"name":"bilibili.png","alternativeText":"bilibili.png","caption":"bilibili.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_bilibili.png","hash":"thumbnail_bilibili_75aa2c0654","ext":".png","mime":"image/png","width":245,"height":73,"size":8.45,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_bilibili_75aa2c0654.png"}},"hash":"bilibili_75aa2c0654","ext":".png","mime":"image/png","size":14.23,"url":"https://official-media.sphere-ex.com/bilibili_75aa2c0654.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:16:47.177Z","updatedAt":"2022-05-13T11:16:47.177Z"}}},"localizations":{"data":[{"id":130,"attributes":{"name":"bilibili","website":null,"createdAt":"2022-05-13T11:17:02.221Z","updatedAt":"2022-05-13T11:17:02.221Z","locale":"en"}}]}}},{"id":131,"attributes":{"name":"yaduo","website":null,"createdAt":"2022-05-13T11:17:14.350Z","updatedAt":"2022-05-13T11:17:25.860Z","locale":"zh","logo":{"data":{"id":252,"attributes":{"name":"ATOUR.png","alternativeText":"ATOUR.png","caption":"ATOUR.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_ATOUR.png","hash":"thumbnail_ATOUR_43065721dc","ext":".png","mime":"image/png","width":245,"height":73,"size":3.9,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_ATOUR_43065721dc.png"}},"hash":"ATOUR_43065721dc","ext":".png","mime":"image/png","size":6.94,"url":"https://official-media.sphere-ex.com/ATOUR_43065721dc.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:16:47.243Z","updatedAt":"2022-05-13T11:16:47.243Z"}}},"localizations":{"data":[{"id":132,"attributes":{"name":"yaduo","website":null,"createdAt":"2022-05-13T11:17:25.836Z","updatedAt":"2022-05-13T11:17:25.836Z","locale":"en"}}]}}},{"id":133,"attributes":{"name":"中移金融科技","website":null,"createdAt":"2022-05-13T11:17:55.006Z","updatedAt":"2022-05-13T11:18:01.693Z","locale":"zh","logo":{"data":{"id":253,"attributes":{"name":"中移金融科技.png","alternativeText":"中移金融科技.png","caption":"中移金融科技.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_中移金融科技.png","hash":"thumbnail__8be830fc07","ext":".png","mime":"image/png","width":245,"height":73,"size":9.7,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__8be830fc07.png"}},"hash":"_8be830fc07","ext":".png","mime":"image/png","size":14.26,"url":"https://official-media.sphere-ex.com/_8be830fc07.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:16:47.279Z","updatedAt":"2022-05-13T11:16:47.279Z"}}},"localizations":{"data":[{"id":134,"attributes":{"name":"中移金融科技","website":null,"createdAt":"2022-05-13T11:18:01.671Z","updatedAt":"2022-05-13T11:18:01.671Z","locale":"en"}}]}}},{"id":135,"attributes":{"name":"转转","website":null,"createdAt":"2022-05-13T11:18:11.675Z","updatedAt":"2022-05-13T11:18:17.664Z","locale":"zh","logo":{"data":{"id":255,"attributes":{"name":"转转.png","alternativeText":"转转.png","caption":"转转.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_转转.png","hash":"thumbnail__03fdcefd4f","ext":".png","mime":"image/png","width":245,"height":73,"size":8.53,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__03fdcefd4f.png"}},"hash":"_03fdcefd4f","ext":".png","mime":"image/png","size":14.8,"url":"https://official-media.sphere-ex.com/_03fdcefd4f.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:16:47.319Z","updatedAt":"2022-05-13T11:16:47.319Z"}}},"localizations":{"data":[{"id":136,"attributes":{"name":"转转","website":null,"createdAt":"2022-05-13T11:18:17.642Z","updatedAt":"2022-05-13T11:18:17.642Z","locale":"en"}}]}}},{"id":137,"attributes":{"name":"CIBN","website":null,"createdAt":"2022-05-13T11:18:31.316Z","updatedAt":"2022-05-13T11:18:42.136Z","locale":"zh","logo":{"data":{"id":254,"attributes":{"name":"CIBN.png","alternativeText":"CIBN.png","caption":"CIBN.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_CIBN.png","hash":"thumbnail_CIBN_f326b4bb6e","ext":".png","mime":"image/png","width":245,"height":73,"size":10.91,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_CIBN_f326b4bb6e.png"}},"hash":"CIBN_f326b4bb6e","ext":".png","mime":"image/png","size":17.42,"url":"https://official-media.sphere-ex.com/CIBN_f326b4bb6e.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:16:47.319Z","updatedAt":"2022-05-13T11:16:47.319Z"}}},"localizations":{"data":[{"id":138,"attributes":{"name":"CIBN","website":null,"createdAt":"2022-05-13T11:18:42.111Z","updatedAt":"2022-05-13T11:18:42.111Z","locale":"en"}}]}}},{"id":139,"attributes":{"name":"TCL","website":null,"createdAt":"2022-05-13T11:19:06.784Z","updatedAt":"2022-05-13T11:19:13.669Z","locale":"zh","logo":{"data":{"id":256,"attributes":{"name":"TCL.png","alternativeText":"TCL.png","caption":"TCL.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_TCL.png","hash":"thumbnail_TCL_7d13dc3257","ext":".png","mime":"image/png","width":245,"height":73,"size":3.37,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_TCL_7d13dc3257.png"}},"hash":"TCL_7d13dc3257","ext":".png","mime":"image/png","size":4.61,"url":"https://official-media.sphere-ex.com/TCL_7d13dc3257.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:18:57.724Z","updatedAt":"2022-05-13T11:18:57.724Z"}}},"localizations":{"data":[{"id":140,"attributes":{"name":"TCL","website":null,"createdAt":"2022-05-13T11:19:13.648Z","updatedAt":"2022-05-13T11:19:13.648Z","locale":"en"}}]}}},{"id":141,"attributes":{"name":"SOUL","website":null,"createdAt":"2022-05-13T11:19:28.899Z","updatedAt":"2022-05-13T11:19:35.488Z","locale":"zh","logo":{"data":{"id":257,"attributes":{"name":"Soul.png","alternativeText":"Soul.png","caption":"Soul.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_Soul.png","hash":"thumbnail_Soul_d4e65f038d","ext":".png","mime":"image/png","width":245,"height":73,"size":7.48,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_Soul_d4e65f038d.png"}},"hash":"Soul_d4e65f038d","ext":".png","mime":"image/png","size":13.21,"url":"https://official-media.sphere-ex.com/Soul_d4e65f038d.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:18:57.791Z","updatedAt":"2022-05-13T11:18:57.791Z"}}},"localizations":{"data":[{"id":142,"attributes":{"name":"SOUL","website":null,"createdAt":"2022-05-13T11:19:35.462Z","updatedAt":"2022-05-13T11:19:35.462Z","locale":"en"}}]}}},{"id":143,"attributes":{"name":"vivo","website":null,"createdAt":"2022-05-13T11:19:50.897Z","updatedAt":"2022-05-13T11:19:57.357Z","locale":"zh","logo":{"data":{"id":258,"attributes":{"name":"vivo.png","alternativeText":"vivo.png","caption":"vivo.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_vivo.png","hash":"thumbnail_vivo_5b4a903f3b","ext":".png","mime":"image/png","width":245,"height":73,"size":7.7,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_vivo_5b4a903f3b.png"}},"hash":"vivo_5b4a903f3b","ext":".png","mime":"image/png","size":15.34,"url":"https://official-media.sphere-ex.com/vivo_5b4a903f3b.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:18:57.796Z","updatedAt":"2022-05-13T11:18:57.796Z"}}},"localizations":{"data":[{"id":144,"attributes":{"name":"vivo","website":null,"createdAt":"2022-05-13T11:19:57.336Z","updatedAt":"2022-05-13T11:19:57.336Z","locale":"en"}}]}}},{"id":145,"attributes":{"name":"OPPO","website":null,"createdAt":"2022-05-13T11:20:10.065Z","updatedAt":"2022-05-13T11:20:15.977Z","locale":"zh","logo":{"data":{"id":259,"attributes":{"name":"OPPO.png","alternativeText":"OPPO.png","caption":"OPPO.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_OPPO.png","hash":"thumbnail_OPPO_2a3767d1ca","ext":".png","mime":"image/png","width":245,"height":73,"size":4.3,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_OPPO_2a3767d1ca.png"}},"hash":"OPPO_2a3767d1ca","ext":".png","mime":"image/png","size":5.57,"url":"https://official-media.sphere-ex.com/OPPO_2a3767d1ca.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:18:57.906Z","updatedAt":"2022-05-13T11:18:57.906Z"}}},"localizations":{"data":[{"id":146,"attributes":{"name":"OPPO","website":null,"createdAt":"2022-05-13T11:20:15.954Z","updatedAt":"2022-05-13T11:20:15.954Z","locale":"en"}}]}}},{"id":147,"attributes":{"name":"Keep","website":null,"createdAt":"2022-05-13T11:20:27.179Z","updatedAt":"2022-05-13T11:20:32.964Z","locale":"zh","logo":{"data":{"id":260,"attributes":{"name":"Keep.png","alternativeText":"Keep.png","caption":"Keep.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_Keep.png","hash":"thumbnail_Keep_49456c2a4a","ext":".png","mime":"image/png","width":245,"height":73,"size":9.44,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_Keep_49456c2a4a.png"}},"hash":"Keep_49456c2a4a","ext":".png","mime":"image/png","size":16.24,"url":"https://official-media.sphere-ex.com/Keep_49456c2a4a.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:18:57.922Z","updatedAt":"2022-05-13T11:18:57.922Z"}}},"localizations":{"data":[{"id":148,"attributes":{"name":"Keep","website":null,"createdAt":"2022-05-13T11:20:32.939Z","updatedAt":"2022-05-13T11:20:32.939Z","locale":"en"}}]}}},{"id":150,"attributes":{"name":"平安银行","website":null,"createdAt":"2022-05-16T02:41:12.800Z","updatedAt":"2022-05-16T02:41:12.800Z","locale":"zh","logo":{"data":{"id":261,"attributes":{"name":"平安银行.png","alternativeText":"平安银行.png","caption":"平安银行.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_平安银行.png","hash":"thumbnail__5e1b99ec05","ext":".png","mime":"image/png","width":245,"height":73,"size":9.97,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__5e1b99ec05.png"}},"hash":"_5e1b99ec05","ext":".png","mime":"image/png","size":13.62,"url":"https://official-media.sphere-ex.com/_5e1b99ec05.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-05-16T02:40:48.928Z","updatedAt":"2022-05-16T02:40:48.928Z"}}},"localizations":{"data":[{"id":149,"attributes":{"name":"平安银行","website":null,"createdAt":"2022-05-16T02:40:57.467Z","updatedAt":"2022-05-16T02:41:12.840Z","locale":"en"}}]}}},{"id":152,"attributes":{"name":"人大金仓","website":null,"createdAt":"2022-10-11T06:04:24.860Z","updatedAt":"2022-10-11T06:04:24.860Z","locale":"zh","logo":{"data":{"id":404,"attributes":{"name":"人大金仓.png","alternativeText":"人大金仓.png","caption":"人大金仓.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_人大金仓.png","hash":"thumbnail__be56b73ca3","ext":".png","mime":"image/png","width":245,"height":73,"size":8.99,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/thumbnail__be56b73ca3.png"}},"hash":"_be56b73ca3","ext":".png","mime":"image/png","size":15.25,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/_be56b73ca3.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-10-11T06:01:39.860Z","updatedAt":"2022-10-11T06:01:39.860Z"}}},"localizations":{"data":[{"id":151,"attributes":{"name":"人大金仓","website":null,"createdAt":"2022-10-11T06:04:09.036Z","updatedAt":"2022-10-11T06:04:24.888Z","locale":"en"}}]}}}],"meta":{"pagination":{"start":0,"limit":100,"total":69}}},"en":{"data":[{"id":14,"attributes":{"name":"58","website":null,"createdAt":"2022-05-12T10:00:08.148Z","updatedAt":"2022-05-12T10:00:08.148Z","locale":"en","logo":{"data":{"id":192,"attributes":{"name":"58同城.png","alternativeText":"58同城.png","caption":"58同城.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_58同城.png","hash":"thumbnail_58_c426aa4c6c","ext":".png","mime":"image/png","width":245,"height":73,"size":7.56,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_58_c426aa4c6c.png"}},"hash":"58_c426aa4c6c","ext":".png","mime":"image/png","size":9.98,"url":"https://official-media.sphere-ex.com/58_c426aa4c6c.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T09:32:56.594Z","updatedAt":"2022-05-12T09:59:17.161Z"}}},"localizations":{"data":[{"id":13,"attributes":{"name":"58","website":null,"createdAt":"2022-05-12T09:59:45.880Z","updatedAt":"2022-05-12T10:00:08.185Z","locale":"zh"}}]}}},{"id":15,"attributes":{"name":"360数科","website":null,"createdAt":"2022-05-12T10:02:13.011Z","updatedAt":"2022-05-12T10:02:26.617Z","locale":"en","logo":{"data":{"id":194,"attributes":{"name":"360 数科.png","alternativeText":"360 数科.png","caption":"360 数科.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_360 数科.png","hash":"thumbnail_360_ba73dda734","ext":".png","mime":"image/png","width":245,"height":73,"size":13.34,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_360_ba73dda734.png"}},"hash":"360_ba73dda734","ext":".png","mime":"image/png","size":24.34,"url":"https://official-media.sphere-ex.com/360_ba73dda734.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T10:02:01.076Z","updatedAt":"2022-05-12T10:02:01.076Z"}}},"localizations":{"data":[{"id":16,"attributes":{"name":"360数科","website":null,"createdAt":"2022-05-12T10:02:26.586Z","updatedAt":"2022-05-12T10:02:26.586Z","locale":"zh"}}]}}},{"id":18,"attributes":{"name":"500PX","website":null,"createdAt":"2022-05-12T10:04:05.674Z","updatedAt":"2022-05-12T10:04:05.674Z","locale":"en","logo":{"data":{"id":195,"attributes":{"name":"500px.png","alternativeText":"500px.png","caption":"500px.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_500px.png","hash":"thumbnail_500px_c2be77bdb0","ext":".png","mime":"image/png","width":245,"height":73,"size":4.56,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_500px_c2be77bdb0.png"}},"hash":"500px_c2be77bdb0","ext":".png","mime":"image/png","size":5.28,"url":"https://official-media.sphere-ex.com/500px_c2be77bdb0.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T10:03:43.369Z","updatedAt":"2022-05-12T10:03:43.369Z"}}},"localizations":{"data":[{"id":17,"attributes":{"name":"500PX","website":null,"createdAt":"2022-05-12T10:03:56.480Z","updatedAt":"2022-05-12T10:04:05.703Z","locale":"zh"}}]}}},{"id":20,"attributes":{"name":"爱番番","website":null,"createdAt":"2022-05-12T10:05:05.682Z","updatedAt":"2022-05-12T10:05:05.682Z","locale":"en","logo":{"data":{"id":196,"attributes":{"name":"爱番番.png","alternativeText":"爱番番.png","caption":"爱番番.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_爱番番.png","hash":"thumbnail__2ec44848c9","ext":".png","mime":"image/png","width":245,"height":73,"size":7.15,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__2ec44848c9.png"}},"hash":"_2ec44848c9","ext":".png","mime":"image/png","size":13.36,"url":"https://official-media.sphere-ex.com/_2ec44848c9.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T10:04:35.351Z","updatedAt":"2022-05-12T10:04:35.351Z"}}},"localizations":{"data":[{"id":19,"attributes":{"name":"爱番番","website":null,"createdAt":"2022-05-12T10:04:52.902Z","updatedAt":"2022-05-12T10:05:05.715Z","locale":"zh"}}]}}},{"id":22,"attributes":{"name":"爱奇艺","website":null,"createdAt":"2022-05-12T10:05:39.037Z","updatedAt":"2022-05-12T10:05:39.037Z","locale":"en","logo":{"data":{"id":197,"attributes":{"name":"爱奇艺.png","alternativeText":"爱奇艺.png","caption":"爱奇艺.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_爱奇艺.png","hash":"thumbnail__ff2f4ac641","ext":".png","mime":"image/png","width":245,"height":73,"size":11.2,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__ff2f4ac641.png"}},"hash":"_ff2f4ac641","ext":".png","mime":"image/png","size":19.87,"url":"https://official-media.sphere-ex.com/_ff2f4ac641.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T10:05:20.081Z","updatedAt":"2022-05-12T10:05:20.081Z"}}},"localizations":{"data":[{"id":21,"attributes":{"name":"爱奇艺","website":null,"createdAt":"2022-05-12T10:05:29.353Z","updatedAt":"2022-05-12T10:05:39.064Z","locale":"zh"}}]}}},{"id":23,"attributes":{"name":"百果园","website":null,"createdAt":"2022-05-12T10:09:23.967Z","updatedAt":"2022-05-12T10:09:39.612Z","locale":"en","logo":{"data":{"id":198,"attributes":{"name":"百果园.png","alternativeText":"百果园.png","caption":"百果园.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_百果园.png","hash":"thumbnail__b5aa05467d","ext":".png","mime":"image/png","width":245,"height":73,"size":8.48,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__b5aa05467d.png"}},"hash":"_b5aa05467d","ext":".png","mime":"image/png","size":15.35,"url":"https://official-media.sphere-ex.com/_b5aa05467d.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T10:09:15.192Z","updatedAt":"2022-05-12T10:09:15.192Z"}}},"localizations":{"data":[{"id":24,"attributes":{"name":"百果园","website":null,"createdAt":"2022-05-12T10:09:39.577Z","updatedAt":"2022-05-12T10:09:39.577Z","locale":"zh"}}]}}},{"id":26,"attributes":{"name":"编程猫","website":null,"createdAt":"2022-05-12T10:10:32.541Z","updatedAt":"2022-05-12T10:10:32.541Z","locale":"en","logo":{"data":{"id":199,"attributes":{"name":"编程猫.png","alternativeText":"编程猫.png","caption":"编程猫.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_编程猫.png","hash":"thumbnail__8a3da21399","ext":".png","mime":"image/png","width":245,"height":73,"size":6.46,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__8a3da21399.png"}},"hash":"_8a3da21399","ext":".png","mime":"image/png","size":9.01,"url":"https://official-media.sphere-ex.com/_8a3da21399.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T10:10:12.303Z","updatedAt":"2022-05-12T10:10:12.303Z"}}},"localizations":{"data":[{"id":25,"attributes":{"name":"编程猫","website":null,"createdAt":"2022-05-12T10:10:23.647Z","updatedAt":"2022-05-12T10:10:32.569Z","locale":"zh"}}]}}},{"id":28,"attributes":{"name":"天威诚信","website":null,"createdAt":"2022-05-12T10:11:06.381Z","updatedAt":"2022-05-12T10:11:06.381Z","locale":"en","logo":{"data":{"id":200,"attributes":{"name":"诚信天威.png","alternativeText":"诚信天威.png","caption":"诚信天威.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_诚信天威.png","hash":"thumbnail__bd7c360f7e","ext":".png","mime":"image/png","width":245,"height":73,"size":7.26,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__bd7c360f7e.png"}},"hash":"_bd7c360f7e","ext":".png","mime":"image/png","size":13.57,"url":"https://official-media.sphere-ex.com/_bd7c360f7e.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T10:10:47.908Z","updatedAt":"2022-05-12T10:10:47.908Z"}}},"localizations":{"data":[{"id":27,"attributes":{"name":"天威诚信","website":null,"createdAt":"2022-05-12T10:10:58.220Z","updatedAt":"2022-05-12T10:11:06.416Z","locale":"zh"}}]}}},{"id":29,"attributes":{"name":"大搜车","website":null,"createdAt":"2022-05-12T11:05:00.232Z","updatedAt":"2022-05-12T11:05:07.849Z","locale":"en","logo":{"data":{"id":201,"attributes":{"name":"大搜车.png","alternativeText":"大搜车.png","caption":"大搜车.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_大搜车.png","hash":"thumbnail__1e1776bd45","ext":".png","mime":"image/png","width":245,"height":73,"size":5.03,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__1e1776bd45.png"}},"hash":"_1e1776bd45","ext":".png","mime":"image/png","size":7.47,"url":"https://official-media.sphere-ex.com/_1e1776bd45.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:04:51.089Z","updatedAt":"2022-05-12T11:04:51.089Z"}}},"localizations":{"data":[{"id":30,"attributes":{"name":"大搜车","website":null,"createdAt":"2022-05-12T11:05:07.827Z","updatedAt":"2022-05-12T11:05:07.827Z","locale":"zh"}}]}}},{"id":32,"attributes":{"name":"当当","website":null,"createdAt":"2022-05-12T11:05:45.489Z","updatedAt":"2022-05-12T11:05:45.489Z","locale":"en","logo":{"data":{"id":202,"attributes":{"name":"当当.png","alternativeText":"当当.png","caption":"当当.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_当当.png","hash":"thumbnail__454b261869","ext":".png","mime":"image/png","width":245,"height":73,"size":9.01,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__454b261869.png"}},"hash":"_454b261869","ext":".png","mime":"image/png","size":15.73,"url":"https://official-media.sphere-ex.com/_454b261869.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:05:27.557Z","updatedAt":"2022-05-12T11:05:27.557Z"}}},"localizations":{"data":[{"id":31,"attributes":{"name":"当当","website":null,"createdAt":"2022-05-12T11:05:35.672Z","updatedAt":"2022-05-12T11:05:45.516Z","locale":"zh"}}]}}},{"id":36,"attributes":{"name":"丁香园","website":null,"createdAt":"2022-05-12T11:07:02.905Z","updatedAt":"2022-05-12T11:07:02.905Z","locale":"en","logo":{"data":{"id":205,"attributes":{"name":"丁香园.png","alternativeText":"丁香园.png","caption":"丁香园.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_丁香园.png","hash":"thumbnail__7baf317e28","ext":".png","mime":"image/png","width":245,"height":73,"size":6.46,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__7baf317e28.png"}},"hash":"_7baf317e28","ext":".png","mime":"image/png","size":9.79,"url":"https://official-media.sphere-ex.com/_7baf317e28.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:06:53.350Z","updatedAt":"2022-05-12T11:06:53.350Z"}}},"localizations":{"data":[{"id":35,"attributes":{"name":"丁香园","website":null,"createdAt":"2022-05-12T11:06:42.076Z","updatedAt":"2022-05-12T11:07:02.931Z","locale":"zh"}}]}}},{"id":38,"attributes":{"name":"得物","website":null,"createdAt":"2022-05-12T11:08:25.812Z","updatedAt":"2022-05-12T11:08:25.812Z","locale":"en","logo":{"data":{"id":203,"attributes":{"name":"得物.png","alternativeText":"得物.png","caption":"得物.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_得物.png","hash":"thumbnail__3625fb8574","ext":".png","mime":"image/png","width":245,"height":73,"size":3.52,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__3625fb8574.png"}},"hash":"_3625fb8574","ext":".png","mime":"image/png","size":3.77,"url":"https://official-media.sphere-ex.com/_3625fb8574.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:06:06.032Z","updatedAt":"2022-05-12T11:06:06.032Z"}}},"localizations":{"data":[{"id":37,"attributes":{"name":"得物","website":null,"createdAt":"2022-05-12T11:08:13.051Z","updatedAt":"2022-05-12T11:08:25.837Z","locale":"zh"}}]}}},{"id":39,"attributes":{"name":"叮当快药","website":null,"createdAt":"2022-05-12T11:09:30.322Z","updatedAt":"2022-05-12T11:09:38.381Z","locale":"en","logo":{"data":{"id":206,"attributes":{"name":"叮当快药.png","alternativeText":"叮当快药.png","caption":"叮当快药.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_叮当快药.png","hash":"thumbnail__0f02ae8949","ext":".png","mime":"image/png","width":245,"height":73,"size":7.88,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__0f02ae8949.png"}},"hash":"_0f02ae8949","ext":".png","mime":"image/png","size":14.81,"url":"https://official-media.sphere-ex.com/_0f02ae8949.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:09:20.787Z","updatedAt":"2022-05-12T11:09:20.787Z"}}},"localizations":{"data":[{"id":40,"attributes":{"name":"叮当快药","website":null,"createdAt":"2022-05-12T11:09:38.353Z","updatedAt":"2022-05-12T11:09:38.353Z","locale":"zh"}}]}}},{"id":42,"attributes":{"name":"斗鱼","website":null,"createdAt":"2022-05-12T11:10:22.593Z","updatedAt":"2022-05-12T11:10:22.593Z","locale":"en","logo":{"data":{"id":207,"attributes":{"name":"斗鱼.png","alternativeText":"斗鱼.png","caption":"斗鱼.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_斗鱼.png","hash":"thumbnail__50476865dc","ext":".png","mime":"image/png","width":245,"height":73,"size":11.3,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__50476865dc.png"}},"hash":"_50476865dc","ext":".png","mime":"image/png","size":22.41,"url":"https://official-media.sphere-ex.com/_50476865dc.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:10:03.411Z","updatedAt":"2022-05-12T11:10:03.411Z"}}},"localizations":{"data":[{"id":41,"attributes":{"name":"斗鱼","website":null,"createdAt":"2022-05-12T11:10:11.928Z","updatedAt":"2022-05-12T11:10:22.618Z","locale":"zh"}}]}}},{"id":44,"attributes":{"name":"公牛","website":null,"createdAt":"2022-05-12T11:10:56.977Z","updatedAt":"2022-05-12T11:10:56.977Z","locale":"en","logo":{"data":{"id":208,"attributes":{"name":"公牛.png","alternativeText":"公牛.png","caption":"公牛.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_公牛.png","hash":"thumbnail__f5c6be9842","ext":".png","mime":"image/png","width":245,"height":73,"size":6.26,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__f5c6be9842.png"}},"hash":"_f5c6be9842","ext":".png","mime":"image/png","size":10.15,"url":"https://official-media.sphere-ex.com/_f5c6be9842.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:10:36.578Z","updatedAt":"2022-05-12T11:10:36.578Z"}}},"localizations":{"data":[{"id":43,"attributes":{"name":"公牛","website":null,"createdAt":"2022-05-12T11:10:47.512Z","updatedAt":"2022-05-12T11:10:57.005Z","locale":"zh"}}]}}},{"id":46,"attributes":{"name":"海典","website":null,"createdAt":"2022-05-12T11:11:45.328Z","updatedAt":"2022-05-12T11:14:58.768Z","locale":"en","logo":{"data":{"id":209,"attributes":{"name":"海典.png","alternativeText":"海典.png","caption":"海典.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_海典.png","hash":"thumbnail__ce8f7e0cc1","ext":".png","mime":"image/png","width":245,"height":73,"size":8.02,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__ce8f7e0cc1.png"}},"hash":"_ce8f7e0cc1","ext":".png","mime":"image/png","size":10.81,"url":"https://official-media.sphere-ex.com/_ce8f7e0cc1.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:11:18.684Z","updatedAt":"2022-05-12T11:11:18.684Z"}}},"localizations":{"data":[{"id":45,"attributes":{"name":"海典","website":null,"createdAt":"2022-05-12T11:11:32.952Z","updatedAt":"2022-05-12T11:14:54.887Z","locale":"zh"}}]}}},{"id":48,"attributes":{"name":"广联达","website":null,"createdAt":"2022-05-12T11:12:09.243Z","updatedAt":"2022-05-12T11:12:09.243Z","locale":"en","logo":{"data":{"id":210,"attributes":{"name":"广联达.png","alternativeText":"广联达.png","caption":"广联达.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_广联达.png","hash":"thumbnail__5b3f63fe0c","ext":".png","mime":"image/png","width":245,"height":73,"size":11.9,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__5b3f63fe0c.png"}},"hash":"_5b3f63fe0c","ext":".png","mime":"image/png","size":24.1,"url":"https://official-media.sphere-ex.com/_5b3f63fe0c.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:11:18.691Z","updatedAt":"2022-05-12T11:11:18.691Z"}}},"localizations":{"data":[{"id":47,"attributes":{"name":"广联达","website":null,"createdAt":"2022-05-12T11:11:59.223Z","updatedAt":"2022-05-12T11:12:09.270Z","locale":"zh"}}]}}},{"id":50,"attributes":{"name":"海尔消费金融","website":null,"createdAt":"2022-05-12T11:12:39.776Z","updatedAt":"2022-05-12T11:12:39.776Z","locale":"en","logo":{"data":{"id":211,"attributes":{"name":"海尔.png","alternativeText":"海尔.png","caption":"海尔.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_海尔.png","hash":"thumbnail__21ad0931e1","ext":".png","mime":"image/png","width":245,"height":73,"size":8.32,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__21ad0931e1.png"}},"hash":"_21ad0931e1","ext":".png","mime":"image/png","size":13.4,"url":"https://official-media.sphere-ex.com/_21ad0931e1.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:11:18.777Z","updatedAt":"2022-05-12T11:11:18.777Z"}}},"localizations":{"data":[{"id":49,"attributes":{"name":"海尔消费金融","website":null,"createdAt":"2022-05-12T11:12:29.224Z","updatedAt":"2022-05-12T11:12:39.803Z","locale":"zh"}}]}}},{"id":52,"attributes":{"name":"瓜子二手车","website":null,"createdAt":"2022-05-12T11:13:19.919Z","updatedAt":"2022-05-12T11:13:19.919Z","locale":"en","logo":{"data":{"id":212,"attributes":{"name":"瓜子.png","alternativeText":"瓜子.png","caption":"瓜子.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_瓜子.png","hash":"thumbnail__ac83d8efd6","ext":".png","mime":"image/png","width":245,"height":73,"size":11.82,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__ac83d8efd6.png"}},"hash":"_ac83d8efd6","ext":".png","mime":"image/png","size":22.1,"url":"https://official-media.sphere-ex.com/_ac83d8efd6.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:11:18.862Z","updatedAt":"2022-05-12T11:11:18.862Z"}}},"localizations":{"data":[{"id":51,"attributes":{"name":"瓜子二手车","website":null,"createdAt":"2022-05-12T11:13:02.908Z","updatedAt":"2022-05-12T11:13:19.943Z","locale":"zh"}}]}}},{"id":54,"attributes":{"name":"怪兽充电","website":null,"createdAt":"2022-05-12T11:13:46.662Z","updatedAt":"2022-05-12T11:13:46.662Z","locale":"en","logo":{"data":{"id":213,"attributes":{"name":"怪兽充电.png","alternativeText":"怪兽充电.png","caption":"怪兽充电.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_怪兽充电.png","hash":"thumbnail__a71d67b0e7","ext":".png","mime":"image/png","width":245,"height":73,"size":8.74,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__a71d67b0e7.png"}},"hash":"_a71d67b0e7","ext":".png","mime":"image/png","size":15.71,"url":"https://official-media.sphere-ex.com/_a71d67b0e7.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:11:18.863Z","updatedAt":"2022-05-12T11:11:18.863Z"}}},"localizations":{"data":[{"id":53,"attributes":{"name":"怪兽充电","website":null,"createdAt":"2022-05-12T11:13:38.051Z","updatedAt":"2022-05-12T11:13:46.687Z","locale":"zh"}}]}}},{"id":56,"attributes":{"name":"海康","website":null,"createdAt":"2022-05-12T11:14:35.811Z","updatedAt":"2022-05-12T11:14:35.811Z","locale":"en","logo":{"data":{"id":214,"attributes":{"name":"海康威视.png","alternativeText":"海康威视.png","caption":"海康威视.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_海康威视.png","hash":"thumbnail__3fd9901f86","ext":".png","mime":"image/png","width":245,"height":73,"size":6.17,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__3fd9901f86.png"}},"hash":"_3fd9901f86","ext":".png","mime":"image/png","size":8.15,"url":"https://official-media.sphere-ex.com/_3fd9901f86.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:14:12.210Z","updatedAt":"2022-05-12T11:14:12.210Z"}}},"localizations":{"data":[{"id":55,"attributes":{"name":"海康","website":null,"createdAt":"2022-05-12T11:14:20.838Z","updatedAt":"2022-05-12T11:14:35.835Z","locale":"zh"}}]}}},{"id":58,"attributes":{"name":"好未来","website":null,"createdAt":"2022-05-12T11:16:04.452Z","updatedAt":"2022-05-12T11:16:04.452Z","locale":"en","logo":{"data":{"id":215,"attributes":{"name":"好未来.png","alternativeText":"好未来.png","caption":"好未来.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_好未来.png","hash":"thumbnail__d93086d051","ext":".png","mime":"image/png","width":245,"height":73,"size":10.17,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__d93086d051.png"}},"hash":"_d93086d051","ext":".png","mime":"image/png","size":20.34,"url":"https://official-media.sphere-ex.com/_d93086d051.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:15:46.920Z","updatedAt":"2022-05-12T11:15:46.920Z"}}},"localizations":{"data":[{"id":57,"attributes":{"name":"好未来","website":null,"createdAt":"2022-05-12T11:15:53.432Z","updatedAt":"2022-05-12T11:16:04.478Z","locale":"zh"}}]}}},{"id":60,"attributes":{"name":"欢聚时代","website":null,"createdAt":"2022-05-12T11:16:27.223Z","updatedAt":"2022-05-12T11:16:27.223Z","locale":"en","logo":{"data":{"id":216,"attributes":{"name":"欢聚时代.png","alternativeText":"欢聚时代.png","caption":"欢聚时代.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_欢聚时代.png","hash":"thumbnail__b286bdb96d","ext":".png","mime":"image/png","width":245,"height":73,"size":12.22,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__b286bdb96d.png"}},"hash":"_b286bdb96d","ext":".png","mime":"image/png","size":22.36,"url":"https://official-media.sphere-ex.com/_b286bdb96d.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:16:14.879Z","updatedAt":"2022-05-12T11:16:14.879Z"}}},"localizations":{"data":[{"id":59,"attributes":{"name":"欢聚时代","website":null,"createdAt":"2022-05-12T11:16:21.597Z","updatedAt":"2022-05-12T11:16:27.246Z","locale":"zh"}}]}}},{"id":62,"attributes":{"name":"环球网校","website":null,"createdAt":"2022-05-12T11:17:07.929Z","updatedAt":"2022-05-12T11:17:07.929Z","locale":"en","logo":{"data":{"id":217,"attributes":{"name":"环球网校.png","alternativeText":"环球网校.png","caption":"环球网校.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_环球网校.png","hash":"thumbnail__57f58a8c7b","ext":".png","mime":"image/png","width":245,"height":73,"size":8.77,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__57f58a8c7b.png"}},"hash":"_57f58a8c7b","ext":".png","mime":"image/png","size":15.82,"url":"https://official-media.sphere-ex.com/_57f58a8c7b.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:16:46.576Z","updatedAt":"2022-05-12T11:16:46.576Z"}}},"localizations":{"data":[{"id":61,"attributes":{"name":"环球网校","website":null,"createdAt":"2022-05-12T11:16:58.746Z","updatedAt":"2022-05-12T11:17:07.952Z","locale":"zh"}}]}}},{"id":64,"attributes":{"name":"货车帮","website":null,"createdAt":"2022-05-12T11:17:37.845Z","updatedAt":"2022-05-12T11:17:37.845Z","locale":"en","logo":{"data":{"id":218,"attributes":{"name":"货车帮.png","alternativeText":"货车帮.png","caption":"货车帮.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_货车帮.png","hash":"thumbnail__d7cbf73239","ext":".png","mime":"image/png","width":245,"height":73,"size":11.36,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__d7cbf73239.png"}},"hash":"_d7cbf73239","ext":".png","mime":"image/png","size":17.92,"url":"https://official-media.sphere-ex.com/_d7cbf73239.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:17:23.196Z","updatedAt":"2022-05-12T11:17:23.196Z"}}},"localizations":{"data":[{"id":63,"attributes":{"name":"货车帮","website":null,"createdAt":"2022-05-12T11:17:31.439Z","updatedAt":"2022-05-12T11:17:37.868Z","locale":"zh"}}]}}},{"id":66,"attributes":{"name":"家视天下","website":null,"createdAt":"2022-05-12T11:18:16.566Z","updatedAt":"2022-05-12T11:18:16.566Z","locale":"en","logo":{"data":{"id":219,"attributes":{"name":"家视天下.png","alternativeText":"家视天下.png","caption":"家视天下.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_家视天下.png","hash":"thumbnail__69bf270b74","ext":".png","mime":"image/png","width":245,"height":73,"size":5.13,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__69bf270b74.png"}},"hash":"_69bf270b74","ext":".png","mime":"image/png","size":7.15,"url":"https://official-media.sphere-ex.com/_69bf270b74.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:17:54.407Z","updatedAt":"2022-05-12T11:17:54.407Z"}}},"localizations":{"data":[{"id":65,"attributes":{"name":"家视天下","website":null,"createdAt":"2022-05-12T11:18:06.319Z","updatedAt":"2022-05-12T11:18:16.588Z","locale":"zh"}}]}}},{"id":68,"attributes":{"name":"建信金科","website":null,"createdAt":"2022-05-12T11:18:49.234Z","updatedAt":"2022-05-12T11:18:49.234Z","locale":"en","logo":{"data":{"id":220,"attributes":{"name":"建信金科.png","alternativeText":"建信金科.png","caption":"建信金科.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_建信金科.png","hash":"thumbnail__6b2c67ba48","ext":".png","mime":"image/png","width":245,"height":73,"size":9.45,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__6b2c67ba48.png"}},"hash":"_6b2c67ba48","ext":".png","mime":"image/png","size":15.25,"url":"https://official-media.sphere-ex.com/_6b2c67ba48.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:18:33.807Z","updatedAt":"2022-05-12T11:18:33.807Z"}}},"localizations":{"data":[{"id":67,"attributes":{"name":"建信金科","website":null,"createdAt":"2022-05-12T11:18:42.863Z","updatedAt":"2022-05-12T11:18:49.256Z","locale":"zh"}}]}}},{"id":70,"attributes":{"name":"京东科技","website":null,"createdAt":"2022-05-12T11:19:17.074Z","updatedAt":"2022-05-12T11:19:17.074Z","locale":"en","logo":{"data":{"id":221,"attributes":{"name":"京东科技.png","alternativeText":"京东科技.png","caption":"京东科技.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_京东科技.png","hash":"thumbnail__36be828c9c","ext":".png","mime":"image/png","width":245,"height":73,"size":7.44,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__36be828c9c.png"}},"hash":"_36be828c9c","ext":".png","mime":"image/png","size":11.23,"url":"https://official-media.sphere-ex.com/_36be828c9c.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:19:02.366Z","updatedAt":"2022-05-12T11:19:02.366Z"}}},"localizations":{"data":[{"id":69,"attributes":{"name":"京东科技","website":null,"createdAt":"2022-05-12T11:19:09.045Z","updatedAt":"2022-05-12T11:19:17.099Z","locale":"zh"}}]}}},{"id":72,"attributes":{"name":"京东物流","website":null,"createdAt":"2022-05-12T11:19:48.106Z","updatedAt":"2022-05-12T11:19:48.106Z","locale":"en","logo":{"data":{"id":222,"attributes":{"name":"京东物流.png","alternativeText":"京东物流.png","caption":"京东物流.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_京东物流.png","hash":"thumbnail__20b318c172","ext":".png","mime":"image/png","width":245,"height":73,"size":6.21,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__20b318c172.png"}},"hash":"_20b318c172","ext":".png","mime":"image/png","size":5.83,"url":"https://official-media.sphere-ex.com/_20b318c172.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:19:28.897Z","updatedAt":"2022-05-12T11:19:28.897Z"}}},"localizations":{"data":[{"id":71,"attributes":{"name":"京东物流","website":null,"createdAt":"2022-05-12T11:19:36.772Z","updatedAt":"2022-05-12T11:19:48.204Z","locale":"zh"}}]}}},{"id":74,"attributes":{"name":"玖富集团","website":null,"createdAt":"2022-05-12T11:20:21.242Z","updatedAt":"2022-05-12T11:20:21.242Z","locale":"en","logo":{"data":{"id":223,"attributes":{"name":"玖富集团.png","alternativeText":"玖富集团.png","caption":"玖富集团.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_玖富集团.png","hash":"thumbnail__a9a3ec5822","ext":".png","mime":"image/png","width":245,"height":73,"size":7.26,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__a9a3ec5822.png"}},"hash":"_a9a3ec5822","ext":".png","mime":"image/png","size":11.86,"url":"https://official-media.sphere-ex.com/_a9a3ec5822.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:20:00.653Z","updatedAt":"2022-05-12T11:20:00.653Z"}}},"localizations":{"data":[{"id":73,"attributes":{"name":"玖富集团","website":null,"createdAt":"2022-05-12T11:20:14.253Z","updatedAt":"2022-05-12T11:20:21.264Z","locale":"zh"}}]}}},{"id":76,"attributes":{"name":"科大讯飞","website":null,"createdAt":"2022-05-12T11:20:45.826Z","updatedAt":"2022-05-12T11:20:45.826Z","locale":"en","logo":{"data":{"id":224,"attributes":{"name":"科大讯飞.png","alternativeText":"科大讯飞.png","caption":"科大讯飞.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_科大讯飞.png","hash":"thumbnail__840f032a53","ext":".png","mime":"image/png","width":245,"height":73,"size":8.73,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__840f032a53.png"}},"hash":"_840f032a53","ext":".png","mime":"image/png","size":12.8,"url":"https://official-media.sphere-ex.com/_840f032a53.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:20:32.329Z","updatedAt":"2022-05-12T11:20:32.329Z"}}},"localizations":{"data":[{"id":75,"attributes":{"name":"科大讯飞","website":null,"createdAt":"2022-05-12T11:20:39.898Z","updatedAt":"2022-05-12T11:20:45.857Z","locale":"zh"}}]}}},{"id":78,"attributes":{"name":"客如云","website":null,"createdAt":"2022-05-12T11:21:12.112Z","updatedAt":"2022-05-12T11:21:12.112Z","locale":"en","logo":{"data":{"id":225,"attributes":{"name":"客如云.png","alternativeText":"客如云.png","caption":"客如云.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_客如云.png","hash":"thumbnail__9539ee7440","ext":".png","mime":"image/png","width":245,"height":73,"size":5.78,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__9539ee7440.png"}},"hash":"_9539ee7440","ext":".png","mime":"image/png","size":4.81,"url":"https://official-media.sphere-ex.com/_9539ee7440.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:20:57.667Z","updatedAt":"2022-05-12T11:20:57.667Z"}}},"localizations":{"data":[{"id":77,"attributes":{"name":"客如云","website":null,"createdAt":"2022-05-12T11:21:06.372Z","updatedAt":"2022-05-12T11:21:12.174Z","locale":"zh"}}]}}},{"id":80,"attributes":{"name":"马上消费","website":null,"createdAt":"2022-05-12T11:22:06.877Z","updatedAt":"2022-05-12T11:22:06.877Z","locale":"en","logo":{"data":{"id":226,"attributes":{"name":"马上消费.png","alternativeText":"马上消费.png","caption":"马上消费.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_马上消费.png","hash":"thumbnail__96b249b440","ext":".png","mime":"image/png","width":245,"height":73,"size":7.5,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__96b249b440.png"}},"hash":"_96b249b440","ext":".png","mime":"image/png","size":11.76,"url":"https://official-media.sphere-ex.com/_96b249b440.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:21:41.147Z","updatedAt":"2022-05-12T11:21:41.147Z"}}},"localizations":{"data":[{"id":79,"attributes":{"name":"马上消费","website":null,"createdAt":"2022-05-12T11:21:50.193Z","updatedAt":"2022-05-12T11:22:06.902Z","locale":"zh"}}]}}},{"id":82,"attributes":{"name":"联通云","website":null,"createdAt":"2022-05-12T11:22:36.369Z","updatedAt":"2022-05-12T11:22:36.369Z","locale":"en","logo":{"data":{"id":227,"attributes":{"name":"联通云.png","alternativeText":"联通云.png","caption":"联通云.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_联通云.png","hash":"thumbnail__b834778d2b","ext":".png","mime":"image/png","width":245,"height":73,"size":8.03,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__b834778d2b.png"}},"hash":"_b834778d2b","ext":".png","mime":"image/png","size":14.26,"url":"https://official-media.sphere-ex.com/_b834778d2b.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:21:41.228Z","updatedAt":"2022-05-12T11:21:41.228Z"}}},"localizations":{"data":[{"id":81,"attributes":{"name":"联通云","website":null,"createdAt":"2022-05-12T11:22:25.376Z","updatedAt":"2022-05-12T11:22:36.391Z","locale":"zh"}}]}}},{"id":84,"attributes":{"name":"马蜂窝","website":null,"createdAt":"2022-05-12T11:23:07.661Z","updatedAt":"2022-05-12T11:23:07.661Z","locale":"en","logo":{"data":{"id":228,"attributes":{"name":"马蜂窝.png","alternativeText":"马蜂窝.png","caption":"马蜂窝.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_马蜂窝.png","hash":"thumbnail__3c4a989dc8","ext":".png","mime":"image/png","width":245,"height":73,"size":8.87,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__3c4a989dc8.png"}},"hash":"_3c4a989dc8","ext":".png","mime":"image/png","size":15.92,"url":"https://official-media.sphere-ex.com/_3c4a989dc8.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:21:41.300Z","updatedAt":"2022-05-12T11:21:41.300Z"}}},"localizations":{"data":[{"id":83,"attributes":{"name":"马蜂窝","website":null,"createdAt":"2022-05-12T11:22:56.870Z","updatedAt":"2022-05-12T11:23:07.683Z","locale":"zh"}}]}}},{"id":86,"attributes":{"name":"快捷通","website":null,"createdAt":"2022-05-12T11:23:37.789Z","updatedAt":"2022-05-12T11:23:37.789Z","locale":"en","logo":{"data":{"id":229,"attributes":{"name":"快捷通.png","alternativeText":"快捷通.png","caption":"快捷通.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_快捷通.png","hash":"thumbnail__7a0f85a547","ext":".png","mime":"image/png","width":245,"height":73,"size":7.73,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__7a0f85a547.png"}},"hash":"_7a0f85a547","ext":".png","mime":"image/png","size":11.36,"url":"https://official-media.sphere-ex.com/_7a0f85a547.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:21:41.388Z","updatedAt":"2022-05-12T11:21:41.388Z"}}},"localizations":{"data":[{"id":85,"attributes":{"name":"快捷通","website":null,"createdAt":"2022-05-12T11:23:30.037Z","updatedAt":"2022-05-12T11:23:37.812Z","locale":"zh"}}]}}},{"id":88,"attributes":{"name":"理想","website":null,"createdAt":"2022-05-12T11:24:03.499Z","updatedAt":"2022-05-12T11:24:03.499Z","locale":"en","logo":{"data":{"id":230,"attributes":{"name":"理想.png","alternativeText":"理想.png","caption":"理想.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_理想.png","hash":"thumbnail__e076874d2d","ext":".png","mime":"image/png","width":245,"height":73,"size":6.59,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__e076874d2d.png"}},"hash":"_e076874d2d","ext":".png","mime":"image/png","size":11.85,"url":"https://official-media.sphere-ex.com/_e076874d2d.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:21:41.511Z","updatedAt":"2022-05-12T11:21:41.511Z"}}},"localizations":{"data":[{"id":87,"attributes":{"name":"理想","website":null,"createdAt":"2022-05-12T11:23:56.121Z","updatedAt":"2022-05-12T11:24:03.523Z","locale":"zh"}}]}}},{"id":90,"attributes":{"name":"每日优鲜","website":null,"createdAt":"2022-05-12T11:31:06.401Z","updatedAt":"2022-05-12T11:31:06.401Z","locale":"en","logo":{"data":{"id":231,"attributes":{"name":"每日优鲜.png","alternativeText":"每日优鲜.png","caption":"每日优鲜.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_每日优鲜.png","hash":"thumbnail__8049aa0985","ext":".png","mime":"image/png","width":245,"height":73,"size":8.03,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__8049aa0985.png"}},"hash":"_8049aa0985","ext":".png","mime":"image/png","size":12.69,"url":"https://official-media.sphere-ex.com/_8049aa0985.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-12T11:30:49.874Z","updatedAt":"2022-05-12T11:30:49.874Z"}}},"localizations":{"data":[{"id":89,"attributes":{"name":"每日优鲜","website":null,"createdAt":"2022-05-12T11:30:58.535Z","updatedAt":"2022-05-12T11:31:06.422Z","locale":"zh"}}]}}},{"id":91,"attributes":{"name":"中国民生银行","website":null,"createdAt":"2022-05-13T08:53:26.029Z","updatedAt":"2022-05-13T08:53:46.005Z","locale":"en","logo":{"data":{"id":232,"attributes":{"name":"民生银行.png","alternativeText":"民生银行.png","caption":"民生银行.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_民生银行.png","hash":"thumbnail__6fd163ced8","ext":".png","mime":"image/png","width":245,"height":73,"size":12.39,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__6fd163ced8.png"}},"hash":"_6fd163ced8","ext":".png","mime":"image/png","size":23.87,"url":"https://official-media.sphere-ex.com/_6fd163ced8.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:53:14.337Z","updatedAt":"2022-05-13T08:53:14.337Z"}}},"localizations":{"data":[{"id":92,"attributes":{"name":"中国民生银行","website":null,"createdAt":"2022-05-13T08:53:45.974Z","updatedAt":"2022-05-13T08:53:45.974Z","locale":"zh"}}]}}},{"id":94,"attributes":{"name":"搜狐新闻","website":null,"createdAt":"2022-05-13T08:54:28.895Z","updatedAt":"2022-05-13T08:54:28.895Z","locale":"en","logo":{"data":{"id":234,"attributes":{"name":"搜狐新闻.png","alternativeText":"搜狐新闻.png","caption":"搜狐新闻.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_搜狐新闻.png","hash":"thumbnail__f06beae44d","ext":".png","mime":"image/png","width":245,"height":73,"size":8.08,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__f06beae44d.png"}},"hash":"_f06beae44d","ext":".png","mime":"image/png","size":14.43,"url":"https://official-media.sphere-ex.com/_f06beae44d.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:54:12.179Z","updatedAt":"2022-05-13T08:54:12.179Z"}}},"localizations":{"data":[{"id":93,"attributes":{"name":"搜狐新闻","website":null,"createdAt":"2022-05-13T08:54:19.289Z","updatedAt":"2022-05-13T08:54:28.945Z","locale":"zh"}}]}}},{"id":96,"attributes":{"name":"汽车之家","website":null,"createdAt":"2022-05-13T08:54:51.347Z","updatedAt":"2022-05-13T08:54:51.347Z","locale":"en","logo":{"data":{"id":233,"attributes":{"name":"汽车之家.png","alternativeText":"汽车之家.png","caption":"汽车之家.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_汽车之家.png","hash":"thumbnail__52e1c6df2d","ext":".png","mime":"image/png","width":245,"height":73,"size":11.84,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__52e1c6df2d.png"}},"hash":"_52e1c6df2d","ext":".png","mime":"image/png","size":20.7,"url":"https://official-media.sphere-ex.com/_52e1c6df2d.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:54:12.154Z","updatedAt":"2022-05-13T08:54:12.154Z"}}},"localizations":{"data":[{"id":95,"attributes":{"name":"汽车之家","website":null,"createdAt":"2022-05-13T08:54:40.671Z","updatedAt":"2022-05-13T08:54:51.370Z","locale":"zh"}}]}}},{"id":98,"attributes":{"name":"闪送","website":null,"createdAt":"2022-05-13T08:55:10.617Z","updatedAt":"2022-05-13T08:55:10.617Z","locale":"en","logo":{"data":{"id":235,"attributes":{"name":"闪送.png","alternativeText":"闪送.png","caption":"闪送.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_闪送.png","hash":"thumbnail__43705d7b17","ext":".png","mime":"image/png","width":245,"height":73,"size":15.58,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__43705d7b17.png"}},"hash":"_43705d7b17","ext":".png","mime":"image/png","size":31.84,"url":"https://official-media.sphere-ex.com/_43705d7b17.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:54:12.223Z","updatedAt":"2022-05-13T08:54:12.223Z"}}},"localizations":{"data":[{"id":97,"attributes":{"name":"闪送","website":null,"createdAt":"2022-05-13T08:55:04.149Z","updatedAt":"2022-05-13T08:55:10.674Z","locale":"zh"}}]}}},{"id":100,"attributes":{"name":"三只松鼠","website":null,"createdAt":"2022-05-13T08:55:28.254Z","updatedAt":"2022-05-13T08:55:28.254Z","locale":"en","logo":{"data":{"id":236,"attributes":{"name":"三只松鼠.png","alternativeText":"三只松鼠.png","caption":"三只松鼠.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_三只松鼠.png","hash":"thumbnail__8c33dfeb6b","ext":".png","mime":"image/png","width":245,"height":73,"size":11.61,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__8c33dfeb6b.png"}},"hash":"_8c33dfeb6b","ext":".png","mime":"image/png","size":22.67,"url":"https://official-media.sphere-ex.com/_8c33dfeb6b.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:54:12.230Z","updatedAt":"2022-05-13T08:54:12.230Z"}}},"localizations":{"data":[{"id":99,"attributes":{"name":"三只松鼠","website":null,"createdAt":"2022-05-13T08:55:21.083Z","updatedAt":"2022-05-13T08:55:28.282Z","locale":"zh"}}]}}},{"id":102,"attributes":{"name":"首汽约车","website":null,"createdAt":"2022-05-13T08:55:52.546Z","updatedAt":"2022-05-13T08:55:52.546Z","locale":"en","logo":{"data":{"id":237,"attributes":{"name":"首汽约车.png","alternativeText":"首汽约车.png","caption":"首汽约车.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_首汽约车.png","hash":"thumbnail__ca02385dfc","ext":".png","mime":"image/png","width":245,"height":73,"size":10.92,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__ca02385dfc.png"}},"hash":"_ca02385dfc","ext":".png","mime":"image/png","size":21.38,"url":"https://official-media.sphere-ex.com/_ca02385dfc.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:54:12.374Z","updatedAt":"2022-05-13T08:54:12.374Z"}}},"localizations":{"data":[{"id":101,"attributes":{"name":"首汽约车","website":null,"createdAt":"2022-05-13T08:55:44.940Z","updatedAt":"2022-05-13T08:55:52.567Z","locale":"zh"}}]}}},{"id":104,"attributes":{"name":"腾讯音乐人","website":null,"createdAt":"2022-05-13T08:56:37.773Z","updatedAt":"2022-05-13T08:56:37.773Z","locale":"en","logo":{"data":{"id":238,"attributes":{"name":"腾讯音乐人.png","alternativeText":"腾讯音乐人.png","caption":"腾讯音乐人.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_腾讯音乐人.png","hash":"thumbnail__691253dfe6","ext":".png","mime":"image/png","width":245,"height":73,"size":7.17,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__691253dfe6.png"}},"hash":"_691253dfe6","ext":".png","mime":"image/png","size":10.04,"url":"https://official-media.sphere-ex.com/_691253dfe6.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:56:21.308Z","updatedAt":"2022-05-13T08:56:21.308Z"}}},"localizations":{"data":[{"id":103,"attributes":{"name":"腾讯音乐人","website":null,"createdAt":"2022-05-13T08:56:30.799Z","updatedAt":"2022-05-13T08:56:37.803Z","locale":"zh"}}]}}},{"id":106,"attributes":{"name":"豌豆公主","website":null,"createdAt":"2022-05-13T08:56:56.457Z","updatedAt":"2022-05-13T08:56:56.457Z","locale":"en","logo":{"data":{"id":239,"attributes":{"name":"豌豆公主.png","alternativeText":"豌豆公主.png","caption":"豌豆公主.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_豌豆公主.png","hash":"thumbnail__87d194643d","ext":".png","mime":"image/png","width":245,"height":73,"size":6.47,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__87d194643d.png"}},"hash":"_87d194643d","ext":".png","mime":"image/png","size":10.93,"url":"https://official-media.sphere-ex.com/_87d194643d.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:56:21.441Z","updatedAt":"2022-05-13T08:56:21.441Z"}}},"localizations":{"data":[{"id":105,"attributes":{"name":"豌豆公主","website":null,"createdAt":"2022-05-13T08:56:49.892Z","updatedAt":"2022-05-13T08:56:56.478Z","locale":"zh"}}]}}},{"id":109,"attributes":{"name":"特微智能","website":null,"createdAt":"2022-05-13T11:06:26.555Z","updatedAt":"2022-05-13T11:06:26.555Z","locale":"en","logo":{"data":{"id":241,"attributes":{"name":"特微智能.png","alternativeText":"特微智能.png","caption":"特微智能.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_特微智能.png","hash":"thumbnail__a3819c29c7","ext":".png","mime":"image/png","width":245,"height":73,"size":10.14,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__a3819c29c7.png"}},"hash":"_a3819c29c7","ext":".png","mime":"image/png","size":16.2,"url":"https://official-media.sphere-ex.com/_a3819c29c7.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:56:21.461Z","updatedAt":"2022-05-13T08:56:21.461Z"}}},"localizations":{"data":[{"id":108,"attributes":{"name":"特微智能","website":null,"createdAt":"2022-05-13T11:06:01.596Z","updatedAt":"2022-05-13T11:06:26.584Z","locale":"zh"}}]}}},{"id":111,"attributes":{"name":"甜橙金融","website":null,"createdAt":"2022-05-13T11:06:50.620Z","updatedAt":"2022-05-13T11:06:50.620Z","locale":"en","logo":{"data":{"id":242,"attributes":{"name":"甜橙金融.png","alternativeText":"甜橙金融.png","caption":"甜橙金融.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_甜橙金融.png","hash":"thumbnail__448c66753b","ext":".png","mime":"image/png","width":245,"height":73,"size":14.46,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__448c66753b.png"}},"hash":"_448c66753b","ext":".png","mime":"image/png","size":27.59,"url":"https://official-media.sphere-ex.com/_448c66753b.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:56:21.476Z","updatedAt":"2022-05-13T08:56:21.476Z"}}},"localizations":{"data":[{"id":110,"attributes":{"name":"甜橙金融","website":null,"createdAt":"2022-05-13T11:06:40.170Z","updatedAt":"2022-05-13T11:06:50.639Z","locale":"zh"}}]}}},{"id":113,"attributes":{"name":"网易游戏","website":null,"createdAt":"2022-05-13T11:07:27.085Z","updatedAt":"2022-05-13T11:07:27.085Z","locale":"en","logo":{"data":{"id":243,"attributes":{"name":"网易游戏.png","alternativeText":"网易游戏.png","caption":"网易游戏.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_网易游戏.png","hash":"thumbnail__6c9d91cfde","ext":".png","mime":"image/png","width":245,"height":73,"size":14.59,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__6c9d91cfde.png"}},"hash":"_6c9d91cfde","ext":".png","mime":"image/png","size":30.86,"url":"https://official-media.sphere-ex.com/_6c9d91cfde.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:07:10.149Z","updatedAt":"2022-05-13T11:07:10.149Z"}}},"localizations":{"data":[{"id":112,"attributes":{"name":"网易游戏","website":null,"createdAt":"2022-05-13T11:07:17.490Z","updatedAt":"2022-05-13T11:07:27.108Z","locale":"zh"}}]}}},{"id":115,"attributes":{"name":"唯品会","website":null,"createdAt":"2022-05-13T11:07:47.052Z","updatedAt":"2022-05-13T11:07:47.052Z","locale":"en","logo":{"data":{"id":244,"attributes":{"name":"唯品会.png","alternativeText":"唯品会.png","caption":"唯品会.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_唯品会.png","hash":"thumbnail__73c4ab0fa5","ext":".png","mime":"image/png","width":245,"height":73,"size":7.59,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__73c4ab0fa5.png"}},"hash":"_73c4ab0fa5","ext":".png","mime":"image/png","size":9.44,"url":"https://official-media.sphere-ex.com/_73c4ab0fa5.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:07:10.159Z","updatedAt":"2022-05-13T11:07:10.159Z"}}},"localizations":{"data":[{"id":114,"attributes":{"name":"唯品会","website":null,"createdAt":"2022-05-13T11:07:38.182Z","updatedAt":"2022-05-13T11:07:47.075Z","locale":"zh"}}]}}},{"id":117,"attributes":{"name":"携程","website":null,"createdAt":"2022-05-13T11:10:33.475Z","updatedAt":"2022-05-13T11:10:33.475Z","locale":"en","logo":{"data":{"id":245,"attributes":{"name":"携程.png","alternativeText":"携程.png","caption":"携程.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_携程.png","hash":"thumbnail__b86b92e821","ext":".png","mime":"image/png","width":245,"height":73,"size":10.25,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__b86b92e821.png"}},"hash":"_b86b92e821","ext":".png","mime":"image/png","size":17.88,"url":"https://official-media.sphere-ex.com/_b86b92e821.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:10:15.339Z","updatedAt":"2022-05-13T11:10:15.339Z"}}},"localizations":{"data":[{"id":116,"attributes":{"name":"携程","website":null,"createdAt":"2022-05-13T11:10:23.100Z","updatedAt":"2022-05-13T11:10:33.504Z","locale":"zh"}}]}}},{"id":119,"attributes":{"name":"易车","website":null,"createdAt":"2022-05-13T11:13:35.354Z","updatedAt":"2022-05-13T11:13:35.354Z","locale":"en","logo":{"data":{"id":246,"attributes":{"name":"易车.png","alternativeText":"易车.png","caption":"易车.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_易车.png","hash":"thumbnail__e0cabfed8a","ext":".png","mime":"image/png","width":245,"height":73,"size":9.82,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__e0cabfed8a.png"}},"hash":"_e0cabfed8a","ext":".png","mime":"image/png","size":19.88,"url":"https://official-media.sphere-ex.com/_e0cabfed8a.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:13:19.094Z","updatedAt":"2022-05-13T11:13:19.094Z"}}},"localizations":{"data":[{"id":118,"attributes":{"name":"易车","website":null,"createdAt":"2022-05-13T11:13:25.517Z","updatedAt":"2022-05-13T11:13:35.381Z","locale":"zh"}}]}}},{"id":121,"attributes":{"name":"云学堂","website":null,"createdAt":"2022-05-13T11:13:56.105Z","updatedAt":"2022-05-13T11:13:56.105Z","locale":"en","logo":{"data":{"id":247,"attributes":{"name":"云学堂.png","alternativeText":"云学堂.png","caption":"云学堂.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_云学堂.png","hash":"thumbnail__38d46e9c1f","ext":".png","mime":"image/png","width":245,"height":73,"size":9.16,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__38d46e9c1f.png"}},"hash":"_38d46e9c1f","ext":".png","mime":"image/png","size":13.93,"url":"https://official-media.sphere-ex.com/_38d46e9c1f.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:13:19.212Z","updatedAt":"2022-05-13T11:13:19.212Z"}}},"localizations":{"data":[{"id":120,"attributes":{"name":"云学堂","website":null,"createdAt":"2022-05-13T11:13:46.465Z","updatedAt":"2022-05-13T11:13:56.133Z","locale":"zh"}}]}}},{"id":123,"attributes":{"name":"韵达","website":null,"createdAt":"2022-05-13T11:14:30.735Z","updatedAt":"2022-05-13T11:14:30.735Z","locale":"en","logo":{"data":{"id":248,"attributes":{"name":"韵达.png","alternativeText":"韵达.png","caption":"韵达.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_韵达.png","hash":"thumbnail__d887641689","ext":".png","mime":"image/png","width":245,"height":73,"size":5.93,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__d887641689.png"}},"hash":"_d887641689","ext":".png","mime":"image/png","size":5.19,"url":"https://official-media.sphere-ex.com/_d887641689.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:13:19.280Z","updatedAt":"2022-05-13T11:13:19.280Z"}}},"localizations":{"data":[{"id":122,"attributes":{"name":"韵达","website":null,"createdAt":"2022-05-13T11:14:22.755Z","updatedAt":"2022-05-13T11:14:30.758Z","locale":"zh"}}]}}},{"id":124,"attributes":{"name":"天涯社区","website":null,"createdAt":"2022-05-13T11:14:57.164Z","updatedAt":"2022-05-13T11:14:57.164Z","locale":"en","logo":{"data":{"id":240,"attributes":{"name":"天涯社区.png","alternativeText":"天涯社区.png","caption":"天涯社区.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_天涯社区.png","hash":"thumbnail__34fdcd8ea3","ext":".png","mime":"image/png","width":245,"height":73,"size":17.91,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__34fdcd8ea3.png"}},"hash":"_34fdcd8ea3","ext":".png","mime":"image/png","size":36.45,"url":"https://official-media.sphere-ex.com/_34fdcd8ea3.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T08:56:21.445Z","updatedAt":"2022-05-13T08:56:21.445Z"}}},"localizations":{"data":[{"id":107,"attributes":{"name":"天涯社区","website":null,"createdAt":"2022-05-13T08:57:19.133Z","updatedAt":"2022-05-13T11:14:57.190Z","locale":"zh"}}]}}},{"id":126,"attributes":{"name":"元气森林","website":null,"createdAt":"2022-05-13T11:15:29.243Z","updatedAt":"2022-05-13T11:15:29.243Z","locale":"en","logo":{"data":{"id":249,"attributes":{"name":"元气森林.png","alternativeText":"元气森林.png","caption":"元气森林.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_元气森林.png","hash":"thumbnail__13df5999cb","ext":".png","mime":"image/png","width":245,"height":73,"size":5.21,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__13df5999cb.png"}},"hash":"_13df5999cb","ext":".png","mime":"image/png","size":9.91,"url":"https://official-media.sphere-ex.com/_13df5999cb.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:13:19.354Z","updatedAt":"2022-05-13T11:13:19.354Z"}}},"localizations":{"data":[{"id":125,"attributes":{"name":"元气森林","website":null,"createdAt":"2022-05-13T11:15:18.862Z","updatedAt":"2022-05-13T11:15:29.267Z","locale":"zh"}}]}}},{"id":128,"attributes":{"name":"中通快递","website":null,"createdAt":"2022-05-13T11:16:29.062Z","updatedAt":"2022-05-13T11:16:29.062Z","locale":"en","logo":{"data":{"id":250,"attributes":{"name":"中通快递.png","alternativeText":"中通快递.png","caption":"中通快递.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_中通快递.png","hash":"thumbnail__dfc99996ce","ext":".png","mime":"image/png","width":245,"height":73,"size":8.56,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__dfc99996ce.png"}},"hash":"_dfc99996ce","ext":".png","mime":"image/png","size":15.26,"url":"https://official-media.sphere-ex.com/_dfc99996ce.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:16:10.523Z","updatedAt":"2022-05-13T11:16:10.523Z"}}},"localizations":{"data":[{"id":127,"attributes":{"name":"中通快递","website":null,"createdAt":"2022-05-13T11:16:17.997Z","updatedAt":"2022-05-13T11:16:29.091Z","locale":"zh"}}]}}},{"id":130,"attributes":{"name":"bilibili","website":null,"createdAt":"2022-05-13T11:17:02.221Z","updatedAt":"2022-05-13T11:17:02.221Z","locale":"en","logo":{"data":{"id":251,"attributes":{"name":"bilibili.png","alternativeText":"bilibili.png","caption":"bilibili.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_bilibili.png","hash":"thumbnail_bilibili_75aa2c0654","ext":".png","mime":"image/png","width":245,"height":73,"size":8.45,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_bilibili_75aa2c0654.png"}},"hash":"bilibili_75aa2c0654","ext":".png","mime":"image/png","size":14.23,"url":"https://official-media.sphere-ex.com/bilibili_75aa2c0654.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:16:47.177Z","updatedAt":"2022-05-13T11:16:47.177Z"}}},"localizations":{"data":[{"id":129,"attributes":{"name":"bilibili","website":null,"createdAt":"2022-05-13T11:16:55.526Z","updatedAt":"2022-05-13T11:17:02.244Z","locale":"zh"}}]}}},{"id":132,"attributes":{"name":"yaduo","website":null,"createdAt":"2022-05-13T11:17:25.836Z","updatedAt":"2022-05-13T11:17:25.836Z","locale":"en","logo":{"data":{"id":252,"attributes":{"name":"ATOUR.png","alternativeText":"ATOUR.png","caption":"ATOUR.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_ATOUR.png","hash":"thumbnail_ATOUR_43065721dc","ext":".png","mime":"image/png","width":245,"height":73,"size":3.9,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_ATOUR_43065721dc.png"}},"hash":"ATOUR_43065721dc","ext":".png","mime":"image/png","size":6.94,"url":"https://official-media.sphere-ex.com/ATOUR_43065721dc.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:16:47.243Z","updatedAt":"2022-05-13T11:16:47.243Z"}}},"localizations":{"data":[{"id":131,"attributes":{"name":"yaduo","website":null,"createdAt":"2022-05-13T11:17:14.350Z","updatedAt":"2022-05-13T11:17:25.860Z","locale":"zh"}}]}}},{"id":134,"attributes":{"name":"中移金融科技","website":null,"createdAt":"2022-05-13T11:18:01.671Z","updatedAt":"2022-05-13T11:18:01.671Z","locale":"en","logo":{"data":{"id":253,"attributes":{"name":"中移金融科技.png","alternativeText":"中移金融科技.png","caption":"中移金融科技.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_中移金融科技.png","hash":"thumbnail__8be830fc07","ext":".png","mime":"image/png","width":245,"height":73,"size":9.7,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__8be830fc07.png"}},"hash":"_8be830fc07","ext":".png","mime":"image/png","size":14.26,"url":"https://official-media.sphere-ex.com/_8be830fc07.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:16:47.279Z","updatedAt":"2022-05-13T11:16:47.279Z"}}},"localizations":{"data":[{"id":133,"attributes":{"name":"中移金融科技","website":null,"createdAt":"2022-05-13T11:17:55.006Z","updatedAt":"2022-05-13T11:18:01.693Z","locale":"zh"}}]}}},{"id":136,"attributes":{"name":"转转","website":null,"createdAt":"2022-05-13T11:18:17.642Z","updatedAt":"2022-05-13T11:18:17.642Z","locale":"en","logo":{"data":{"id":255,"attributes":{"name":"转转.png","alternativeText":"转转.png","caption":"转转.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_转转.png","hash":"thumbnail__03fdcefd4f","ext":".png","mime":"image/png","width":245,"height":73,"size":8.53,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__03fdcefd4f.png"}},"hash":"_03fdcefd4f","ext":".png","mime":"image/png","size":14.8,"url":"https://official-media.sphere-ex.com/_03fdcefd4f.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:16:47.319Z","updatedAt":"2022-05-13T11:16:47.319Z"}}},"localizations":{"data":[{"id":135,"attributes":{"name":"转转","website":null,"createdAt":"2022-05-13T11:18:11.675Z","updatedAt":"2022-05-13T11:18:17.664Z","locale":"zh"}}]}}},{"id":138,"attributes":{"name":"CIBN","website":null,"createdAt":"2022-05-13T11:18:42.111Z","updatedAt":"2022-05-13T11:18:42.111Z","locale":"en","logo":{"data":{"id":254,"attributes":{"name":"CIBN.png","alternativeText":"CIBN.png","caption":"CIBN.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_CIBN.png","hash":"thumbnail_CIBN_f326b4bb6e","ext":".png","mime":"image/png","width":245,"height":73,"size":10.91,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_CIBN_f326b4bb6e.png"}},"hash":"CIBN_f326b4bb6e","ext":".png","mime":"image/png","size":17.42,"url":"https://official-media.sphere-ex.com/CIBN_f326b4bb6e.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:16:47.319Z","updatedAt":"2022-05-13T11:16:47.319Z"}}},"localizations":{"data":[{"id":137,"attributes":{"name":"CIBN","website":null,"createdAt":"2022-05-13T11:18:31.316Z","updatedAt":"2022-05-13T11:18:42.136Z","locale":"zh"}}]}}},{"id":140,"attributes":{"name":"TCL","website":null,"createdAt":"2022-05-13T11:19:13.648Z","updatedAt":"2022-05-13T11:19:13.648Z","locale":"en","logo":{"data":{"id":256,"attributes":{"name":"TCL.png","alternativeText":"TCL.png","caption":"TCL.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_TCL.png","hash":"thumbnail_TCL_7d13dc3257","ext":".png","mime":"image/png","width":245,"height":73,"size":3.37,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_TCL_7d13dc3257.png"}},"hash":"TCL_7d13dc3257","ext":".png","mime":"image/png","size":4.61,"url":"https://official-media.sphere-ex.com/TCL_7d13dc3257.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:18:57.724Z","updatedAt":"2022-05-13T11:18:57.724Z"}}},"localizations":{"data":[{"id":139,"attributes":{"name":"TCL","website":null,"createdAt":"2022-05-13T11:19:06.784Z","updatedAt":"2022-05-13T11:19:13.669Z","locale":"zh"}}]}}},{"id":142,"attributes":{"name":"SOUL","website":null,"createdAt":"2022-05-13T11:19:35.462Z","updatedAt":"2022-05-13T11:19:35.462Z","locale":"en","logo":{"data":{"id":257,"attributes":{"name":"Soul.png","alternativeText":"Soul.png","caption":"Soul.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_Soul.png","hash":"thumbnail_Soul_d4e65f038d","ext":".png","mime":"image/png","width":245,"height":73,"size":7.48,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_Soul_d4e65f038d.png"}},"hash":"Soul_d4e65f038d","ext":".png","mime":"image/png","size":13.21,"url":"https://official-media.sphere-ex.com/Soul_d4e65f038d.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:18:57.791Z","updatedAt":"2022-05-13T11:18:57.791Z"}}},"localizations":{"data":[{"id":141,"attributes":{"name":"SOUL","website":null,"createdAt":"2022-05-13T11:19:28.899Z","updatedAt":"2022-05-13T11:19:35.488Z","locale":"zh"}}]}}},{"id":144,"attributes":{"name":"vivo","website":null,"createdAt":"2022-05-13T11:19:57.336Z","updatedAt":"2022-05-13T11:19:57.336Z","locale":"en","logo":{"data":{"id":258,"attributes":{"name":"vivo.png","alternativeText":"vivo.png","caption":"vivo.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_vivo.png","hash":"thumbnail_vivo_5b4a903f3b","ext":".png","mime":"image/png","width":245,"height":73,"size":7.7,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_vivo_5b4a903f3b.png"}},"hash":"vivo_5b4a903f3b","ext":".png","mime":"image/png","size":15.34,"url":"https://official-media.sphere-ex.com/vivo_5b4a903f3b.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:18:57.796Z","updatedAt":"2022-05-13T11:18:57.796Z"}}},"localizations":{"data":[{"id":143,"attributes":{"name":"vivo","website":null,"createdAt":"2022-05-13T11:19:50.897Z","updatedAt":"2022-05-13T11:19:57.357Z","locale":"zh"}}]}}},{"id":146,"attributes":{"name":"OPPO","website":null,"createdAt":"2022-05-13T11:20:15.954Z","updatedAt":"2022-05-13T11:20:15.954Z","locale":"en","logo":{"data":{"id":259,"attributes":{"name":"OPPO.png","alternativeText":"OPPO.png","caption":"OPPO.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_OPPO.png","hash":"thumbnail_OPPO_2a3767d1ca","ext":".png","mime":"image/png","width":245,"height":73,"size":4.3,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_OPPO_2a3767d1ca.png"}},"hash":"OPPO_2a3767d1ca","ext":".png","mime":"image/png","size":5.57,"url":"https://official-media.sphere-ex.com/OPPO_2a3767d1ca.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:18:57.906Z","updatedAt":"2022-05-13T11:18:57.906Z"}}},"localizations":{"data":[{"id":145,"attributes":{"name":"OPPO","website":null,"createdAt":"2022-05-13T11:20:10.065Z","updatedAt":"2022-05-13T11:20:15.977Z","locale":"zh"}}]}}},{"id":148,"attributes":{"name":"Keep","website":null,"createdAt":"2022-05-13T11:20:32.939Z","updatedAt":"2022-05-13T11:20:32.939Z","locale":"en","logo":{"data":{"id":260,"attributes":{"name":"Keep.png","alternativeText":"Keep.png","caption":"Keep.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_Keep.png","hash":"thumbnail_Keep_49456c2a4a","ext":".png","mime":"image/png","width":245,"height":73,"size":9.44,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_Keep_49456c2a4a.png"}},"hash":"Keep_49456c2a4a","ext":".png","mime":"image/png","size":16.24,"url":"https://official-media.sphere-ex.com/Keep_49456c2a4a.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-05-13T11:18:57.922Z","updatedAt":"2022-05-13T11:18:57.922Z"}}},"localizations":{"data":[{"id":147,"attributes":{"name":"Keep","website":null,"createdAt":"2022-05-13T11:20:27.179Z","updatedAt":"2022-05-13T11:20:32.964Z","locale":"zh"}}]}}},{"id":149,"attributes":{"name":"平安银行","website":null,"createdAt":"2022-05-16T02:40:57.467Z","updatedAt":"2022-05-16T02:41:12.840Z","locale":"en","logo":{"data":{"id":261,"attributes":{"name":"平安银行.png","alternativeText":"平安银行.png","caption":"平安银行.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_平安银行.png","hash":"thumbnail__5e1b99ec05","ext":".png","mime":"image/png","width":245,"height":73,"size":9.97,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail__5e1b99ec05.png"}},"hash":"_5e1b99ec05","ext":".png","mime":"image/png","size":13.62,"url":"https://official-media.sphere-ex.com/_5e1b99ec05.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-05-16T02:40:48.928Z","updatedAt":"2022-05-16T02:40:48.928Z"}}},"localizations":{"data":[{"id":150,"attributes":{"name":"平安银行","website":null,"createdAt":"2022-05-16T02:41:12.800Z","updatedAt":"2022-05-16T02:41:12.800Z","locale":"zh"}}]}}},{"id":151,"attributes":{"name":"人大金仓","website":null,"createdAt":"2022-10-11T06:04:09.036Z","updatedAt":"2022-10-11T06:04:24.888Z","locale":"en","logo":{"data":{"id":404,"attributes":{"name":"人大金仓.png","alternativeText":"人大金仓.png","caption":"人大金仓.png","width":400,"height":120,"formats":{"thumbnail":{"name":"thumbnail_人大金仓.png","hash":"thumbnail__be56b73ca3","ext":".png","mime":"image/png","width":245,"height":73,"size":8.99,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/thumbnail__be56b73ca3.png"}},"hash":"_be56b73ca3","ext":".png","mime":"image/png","size":15.25,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/_be56b73ca3.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-10-11T06:01:39.860Z","updatedAt":"2022-10-11T06:01:39.860Z"}}},"localizations":{"data":[{"id":152,"attributes":{"name":"人大金仓","website":null,"createdAt":"2022-10-11T06:04:24.860Z","updatedAt":"2022-10-11T06:04:24.860Z","locale":"zh"}}]}}}],"meta":{"pagination":{"start":0,"limit":100,"total":69}}}},"caseStudiesTypes":{"zh":{"data":[{"id":3,"attributes":{"name":"金融","createdAt":"2021-12-11T23:02:27.233Z","updatedAt":"2021-12-11T23:02:50.766Z","publishedAt":"2021-12-11T23:02:50.699Z","locale":"zh","localizations":{"data":[]}}},{"id":4,"attributes":{"name":"电商","createdAt":"2021-12-11T23:02:39.013Z","updatedAt":"2021-12-11T23:02:41.225Z","publishedAt":"2021-12-11T23:02:41.156Z","locale":"zh","localizations":{"data":[]}}}],"meta":{"pagination":{"page":1,"pageSize":25,"pageCount":1,"total":2}}},"en":{"data":[{"id":1,"attributes":{"name":"Finance","createdAt":"2021-12-11T23:01:56.177Z","updatedAt":"2021-12-11T23:01:58.635Z","publishedAt":"2021-12-11T23:01:58.567Z","locale":"en","localizations":{"data":[]}}},{"id":2,"attributes":{"name":"E-commerce","createdAt":"2021-12-11T23:02:08.395Z","updatedAt":"2021-12-11T23:02:11.020Z","publishedAt":"2021-12-11T23:02:10.947Z","locale":"en","localizations":{"data":[]}}},{"id":6,"attributes":{"name":"Media","createdAt":"2022-09-26T10:29:40.567Z","updatedAt":"2022-09-27T06:59:40.008Z","publishedAt":"2022-09-27T06:59:40.006Z","locale":"en","localizations":{"data":[]}}},{"id":7,"attributes":{"name":"Telecom","createdAt":"2022-09-26T10:29:57.244Z","updatedAt":"2022-09-27T06:59:48.158Z","publishedAt":"2022-09-27T06:59:48.156Z","locale":"en","localizations":{"data":[]}}},{"id":9,"attributes":{"name":"Internet","createdAt":"2022-09-26T10:34:03.086Z","updatedAt":"2022-09-27T06:59:34.838Z","publishedAt":"2022-09-27T06:59:34.836Z","locale":"en","localizations":{"data":[]}}},{"id":10,"attributes":{"name":"Software Development","createdAt":"2022-09-26T10:34:53.523Z","updatedAt":"2022-09-27T06:59:43.907Z","publishedAt":"2022-09-27T06:59:43.905Z","locale":"en","localizations":{"data":[]}}}],"meta":{"pagination":{"page":1,"pageSize":25,"pageCount":1,"total":6}}}},"caseStudies":{"zh":{"data":[{"id":49,"attributes":{"feature":false,"title":"某央企｜打造新一代云上数据底座，助力业务推广","content":"## 业务挑战\n根据客户总公司上云整体目标，为推动下一阶段相关业务板块全面推广使用奠定良好基础，需完成项目管理系统上云升级改造工作。\n项目管理系统所使用的技术栈为 Java + MySQL，为满足未来更多项目的云上高效管理，结合当前系统架构，围绕业务、研发及运维三个维度进行了梳理，本次上云升级改造工作需同时达成如下目标。\n1. 提升系统扩展能力\n2. 提高系统资源利用率\n3. 减少查询流程响应时间\n\n## 解决方案\n面向业务构建数据通用接入层，通过 “DBPlusEngine + RDS + 云平台” 打造云上数据底座，提供轻量化分布式数据库解决方案。\n在沿用 MySQL 技术栈基础之上提供分布式计算和存储能力，客户端（Driver）访问模式为业务提供低延时响应支持，服务端（Proxy）访问模式为运维提供便捷维护入口。\n数据分片及分布式事务的支持使应用程序无需修改；全局高可用、多数据库支持及一体化运维能力减轻运维工作。\n\n![20231016-145048.jpeg](https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/20231016_145048_86ccdc01f7.jpeg)\n\n## 客户收益\n- 平滑迁移上线：保持了原有业务逻辑与数据库使用模式，人员、生态及工具均可复用；\n- 避免厂商绑定：为自主创新落地提供必要支撑平台，多数据库支持可避免被指定厂商绑定；\n- 提升扩展能力：计算及存储扩展灵活，新一代数据底座为业务快速增长提供良好扩展能力；\n- 提升系统效能：显著提高云上资源利用率，压力负载均衡，同时业务请求响应时间缩短近 90%。\n\n## 客户感言\n“SphereEx 数据库增强平台具有良好的扩展能力，最重要的是对业务没有入侵。对于不希望改变技术栈，同时有分布式改造升级诉求的业务场景，这样的方案是一个合适的选择。”\n","date":"2023-09-27","author":"代野","excerpt":null,"metaDescription":null,"createdAt":"2023-10-16T06:53:13.035Z","updatedAt":"2023-10-16T06:53:14.701Z","publishedAt":"2023-10-16T06:53:14.698Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":48,"attributes":{"feature":false,"title":"某银行的数据库网关实践","content":" ## 业务挑战\n该银行部分系统采用了 DNS + MySQL 主从的高可用架构，如风控系统。当数据库主节点发生故障时，需通过人工介入切换域名的方式进行重新映射，来完成整体 failover，RTO 为分钟级。\n为进一步改善业务体验，同时满足未来业务发展需要，需要在对业务无入侵前提下，达成如下目标：\n1. 提升业务系统的「可靠性」；\n2. 提高业务系统的「可维护性」；\n3. 增强业务系统的「安全性」；\n4. 增加系统架构的「可扩展性」。\n\n## 设计思路\n为了在现有技术栈的基础上实现可靠性、可维护性、安全性和可扩展性的提升目标，在设计思路上考虑引入数据库网关。通过引入数据库网关，可以为用户提供一个统一的访问接口，充当数据库系统和应用程序之间的桥梁。这样一来，应用程序与数据库之间的交互变得更加简化，同时为架构、研发和运维团队提供了其所关注的能力。\n### 1. 统一接入\n目前行内所使用数据库种类较多，通过数据库网关对分散的数据源进行整合，为用户提供统一的数据库视图和接口，无须再关注底层存储细节。\n### 2. 可用性探测\n可用性探测能力能够主动监测和检测底层数据库系统的可用性，并及时采取相应措施以确保系统的连续可用性。\n### 3. 安全合规\n金融行业对数据的安全性和合规性要求非常高。借助数据库网关为业务系统提供更强大的安全管控能力，如访问控制、数据加密和审计日志等，以确保敏感数据的保护和合规性要求的满足，遵守相关法规和监管要求。\n### 4. 高性能与可伸缩性\n对于金融场景海量数据处理和高并发请求，数据库网关可以提供高性能和可伸缩性的能力，通过读写分离、负载均衡及弹性伸缩等技术，确保业务快速响应及保持良好的扩展性，为用户提供高效的交易处理和良好的体验。\n### 5. 数据分析和业务洞察\n金融机构需定期对大量的数据进行分析，以获取业务洞察、风险评估和决策支持。数据库网关的联邦查询能力可以支持跨系统异构数据的实时访问和查询，通过复杂数据的分析了解客户行为、市场趋势和风险状况，从而优化业务策略和创新产品。\n综上，对于该系统优化升级需求，引入数据库网关是一个升级平滑、投入小且收益高的方案。\n\n## 解决方案\nSphereEx 数据库网关支持多种类型数据库，它面向应用程序和用户提供数据库网关功能，如读写分离、负载均衡、高可用探测、数据库防火墙、访问控制和安全审计等能力，支持如 Oracle、MySQL、PostgreSQL、openGauss、人大金仓及其他国产化数据库。\n在风控系统中，网关的高可用探测能力实现了数据库节点状态感知、确认故障及上线从库全流程切换处理，过程中无需人工介入，达成提升业务系统可靠性的目标。\n![20231016-144927.jpeg](https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/20231016_144927_53ea986f5c.jpeg)\n\n下面是关于故障探测、重试及切换过程的描述。\n1. 监测连接状态：数据库网关能够周期性地探测与底层数据库系统之间的连接状态。通过心跳的验证，网关可以验证与数据库的连接是否正常，确保数据库系统处于可用状态。\n2. 故障检测与重试：当数据库网关在请求数据库时遇到连接异常或错误时，它能够执行故障恢复机制。网关可以自动进行错误处理，例如进行重试操作，恢复数据库连接，尽量避免中断用户的请求。\n3. 确认故障与切换：当数据库网关发现底层数据库系统出现故障或不可用的情况时，它具备自动切换到备用数据库的能力。网关会及时检测到故障，并迅速切换到备用数据库，从而保障系统的连续可用性。\n4. 报警与日志记录：数据库网关可以生成详细的日志记录，包括连接状态、故障信息、切换过程以及其他相关的事件，结合日志内容可通过运维平台发送警报通知，及时获取异常及恢复情况。\n\n总的来说，通过网关可用性探测能力，能够实现对底层数据库系统的主动监测和故障处理，提高系统的可用性和稳定性。这样可以确保系统在面对数据库故障或异常情况时能够快速切换和恢复，并提供可靠的服务给用户。\n\n\n此外，对于系统下一阶段的扩展性要求，SphereEx 数据库网关预留了较好的扩展空间。\n- 读写分离：智能路由读写流量，提升系统整体吞吐量、提升资源利用率，主从延迟感知能力保证数据一致性；\n- 数据库防火墙：通过 SQL 匹配的方式识别并完成风险 SQL 拦截，避免误操作，最大程度降低业务系统风险；\n- 访问控制：精确控制授予每个用户的库级、表级、列级的操作权限，可对接第三方 LDAP；\n- 数据库审计：记录所有执行过的 SQL 语句，便于企业进行 SQL 安全审计操作。\n\n## 客户收益\n通过 SphereEx 数据库网关解决方案，该银行客户面向架构、研发和 DBA 构建起了面向数据库的功能生态，将异构数据库的全局能力统一纳管，应对灵活多变的应用场景。\n1. RTO 由分钟级降至秒级；\n2. 符合金融级业务可用性要求；\n3. 全流程自动化 failover，无需人工介入；\n4. 预留扩展插槽，如读写分离、安全管控等能力。\n\n\n\n\n","date":"2023-08-14","author":"代野","excerpt":null,"metaDescription":null,"createdAt":"2023-10-16T06:50:07.037Z","updatedAt":"2023-10-16T06:50:36.811Z","publishedAt":"2023-10-16T06:50:08.773Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":34,"attributes":{"feature":true,"title":"F6 汽车科技基于 Apache ShardingSphere 的核心业务分库分表实践","content":"## 业务背景\nF6 汽车科技，是一家专注于汽车后市场信息化建设的互联网平台公司，为维修企业开发智慧管理平台。各个维修企业（后文简称商户）之间的数据是相互隔离的，不同商户之间的数据理论上可以存储在不同库不同表。随着公司业务发展迅速，有些表的数据量增长迅速，单表总的数据量达到千万甚至亿的级别，系统越来越难以满足业务的高速发展。另外随着业务发展公司也在对系统进行拆分，按照不同域不同业务拆分成许多微服务，随之也就垂直拆分成了不同的业务库。  \n \n## 业务挑战\n关系型数据库由于单机存储容量、连接数、处理能力都有限，容易成为系统瓶颈。从性能上看，当单表的数据量达到千万以后，由于查询维度较多，即使添加从库、优化索引，做很多操作时性能仍下降严重。此时就要考虑对其进行切分了，切分的目的就在于减少数据库的负担，缩短查询时间。另外单库的连接数有限，如果数据库查询的 QPS 过高，那么就需要通过分库来分担单个数据库的连接压力。从可用性上看，单个数据库如果发生意外，很可能会丢失所有数据，影响所有业务，分库可以隔离故障，减少业务的影响范围。\n \n## 总体方案\n结合公司实际情况我们选择了 ShardingSphere 实现 Sharding。按照公司现有业务逻辑，因此采用商户 ID 作为分片键（ShardingKey），保证一个商户的工单数据，分配在同一个库同一个单表中，避免了多表查询关联的性能损耗，后期分片到多个库时，可以避免跨库事务及跨库 JOIN。\n商户 ID 数据库中的类型为 BIGINT(20)，为了保证后续分库扩容的需求，采用基因法，选取商户 ID 最后两位作为分库基因，按照双倍扩容的规则，可以最大扩展到 64 个库中。剩余位数的值用于分表，分片到 32 个分表中。\n\n规则如下（10545055917999668983 为某商户 ID）：\n\n```\n105450559179996689    83\n分表基因值 % 32    分库基因值 % 1\n```\n\n最后两位 83 用于分库，暂时数据只分片到 f6xxx 一个库中，因此余数为 0，后期数据量增加，扩展至多个库。剩余数值 105450559179996689 用于分表，首次分为 32 个单表，取模余数对应具体的分表下标，为 0~31。\n\n![screenshot-20230321-153539.png](https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/screenshot_20230321_153539_94108c0802.png)\n按照公司系统现状及开发功能能快速迭代分步上线的背景，因此我们制定了先分表再拆库的方案，考虑到分库分表改动影响比较大，因此需要灰度上线，一旦有问题需要快速回滚，不能影响正常业务。具体实施步骤如下：\n### 分表\n（1）工程从 JDBC 切到 Sharding-JDBC 数据源连接方式\n（2）写库业务解耦和代码迁移\n（3）历史数据和增量数据同步\n（4）切分表\n### 分库\n（1）读库业务迁移\n（2）数据迁移\n（3）切读库\n（4）切写库\n\n## 选择 Apache ShardingSphere 的理由\n- 社区活跃，目前已经升级至 5.0 版本，仍处于快速迭代中；\n- 成功应用案例多，京东、当当等大公司广泛应用；\n- 使用简单，Sharding-JDBC 快速集成项目，不需要部署额外服务；\n- 兼容性好，路由至单数据节点，SQL 100% 支持；\n- 性能优异，损耗低，官网有测试结果；\n\n\n","date":"2023-01-02","author":"SphereEx ","excerpt":"汽车互联网平台领域技术实践        ","metaDescription":null,"createdAt":"2023-03-21T07:36:34.241Z","updatedAt":"2024-01-03T09:57:02.591Z","publishedAt":"2023-03-21T07:36:35.817Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":28,"attributes":{"feature":true,"title":"中商惠⺠交易中台架构演进：对 Apache ShardingSphere 的应⽤","content":"> 项目背景\n中商惠民历经⼏次技术迭代，2016 到 2017 年开始 PHP 技术栈转到 Java 技术栈并进⾏微服务化，2018 年开始打造中台。\n2021 年年初完成的交易中台⼀期上线，2022 年 3 ⽉完成⼆期上线。\n\n伴随着业务成长和多样性的变化，要及时响应新需求，同时要避免对现有业务产⽣影响，本着提⾼⼈效，降低成本的原则，中商惠民推出了强中台发展战略。对系统⽽⾔，要降低系统之间的耦合度，增加可扩展性，提取业务共性的同时要保证低延迟。在 2021 年之初完成了对之前订单管理系统（OMS）的重构，推出了以订单的交易过程为核⼼的交易中台项⽬，分为以下四个核⼼模块：订单状态流转系统，订单管理系统，订单履约系统，订单费⽤计算系统。同时，基于业务分层，又拆分为订单查询系统，报表系统，订单管理后台等。\n\n在系统的重构过程中避免技术需求对业务需求产⽣阻碍，上线过程分为两个阶段：应⽤系统拆分和数据拆分。\n\n## 业务挑战\n数据表拆分之后，需要解决数据散列和查询问题。作为⼀家中⼩型企业，⾃研⼀套分库分表的中间件成本过⾼，初期还会⾯临各种踩坑的风险，\n\n## 拆分原则\n以 3 年作为⼀个迭代周期，拆分数量=（（每⽉新增的数据量* 36）+ 历史数据量）/单表数据量上限\n根据我们维护阿⾥云 RDS MySQL 的经验来看，基于在使⽤ Innodb 引擎下，进⾏ DDL 操作时，⼩于 500 万数据量，执⾏时间⼤概⼏⼗秒；⼤于 500 万，⼩于 1000 万数据量，执⾏时间⼤概 500 秒左右；⼤于 1000 万数据量，⼩于 5000 万左右，执⾏时间⼤概 1000 多秒；⼤于 5000 万以上数据量，执⾏时间在 2000 秒以上。\n单表分配上限可以取决于单表操作对业务带来的影响，当然也有⼀些⽅案可以解决 DDL 操作锁表的问题，例如双表同步，切换表名的⽅式可以将影响降低为秒级。\n库表的拆分数量最好是 2 的 N 次⽅，有利于后期做⽔平扩容。\n\n## 解决方案\nShardingSphere 提供了⼀种解决思路。它提出了连接模式（Connection Mode）的概念，将其划分为内存限制模式（MEMORY_STRICTLY）和连接限制模式（CONNECTION_STRICTLY）这两种类型。\n\n### 内存限制模式\n\n使⽤此模式的前提是，ShardingSphere 对⼀次操作所耗费的数据库连接数量不做限制。如果实际执⾏的 SQL 需要对某数据库实例中的 200 张表做操作，则对每张表创建⼀个新的数据库连接，并通过多线程的⽅式并发处理，以达成执⾏效率最⼤化。并且在 SQL 满⾜条件情况下，优先选择流式归并，以防⽌出现内存溢出或避免频繁垃圾回收情况。\n### 连接限制模式\n\n使⽤此模式的前提是，ShardingSphere 严格控制对⼀次操作所耗费的数据库连接数量。如果实际执⾏的 SQL 需要对某数据库实例中的 200 张表做操作，那么只会创建唯⼀的数据库连接，并对其 200 张表串⾏处理。如果⼀次操作中的分⽚散落在不同的数据库，仍然采⽤多线程处理对不同库的操作，但每个库的每次操作仍然只创建⼀个唯⼀的数据库连接。这样即可以防⽌对⼀次请求对数据库连接占⽤过多所带来的问题。该模式始终选择内存归并。\n\n内存限制模式适⽤于 OLAP 操作，可以通过放宽对数据库连接的限制提升系统吞吐量；连接限制模式适⽤于 OLTP 操作，OLTP 通常带有分⽚键，会路由到单⼀的分⽚，因此严格控制数据库连接，以保证在线系统数据库资源能够被更多的应⽤所使⽤，是明智的选择。\n\n我们发现在内存限制模式下，过程中会因为 MySQL 的 innodb 引擎的 cache buffer 加载策略导致操作变为 io 密集型，导致 SQL ⼤量超时，解决此问题的办法就是在不变更数据库资源的情况下我们程序多⼀层处理，如果发现没有分⽚键的时候，先在外置索引中确认⼀下分⽚键，再通过分⽚键来进⾏数据库检索。\n\n## 选择 Apache ShardingSphere 的理由\n\n1. Apache ShardingSphere 功能符合预期，可以解决⽬前的问题并且有丰富的扩展性；\n2. Apache ShardingSphere 社区活跃度⾼，遇到问题会有专⼈及时响应；\n3. 公司采⽤ SpringCloud 技术栈，集成⽅便，成本低；\n4. 性能表现符合预期，完全可以⽀撑现有业务。\n\n## 用户收益\n\n- 性能提升\n通过架构重构，有效控制单表数据量，⼤幅缩减慢 SQL，效率提升近 50%。\n\n- 节省研发资源，降低成本\n引⼊成熟的 Apache ShardingSphere ⽆需重新开发分表组件，降低了研发和踩坑的成本，研发同学只需要集中精⼒处理业务问题。\n\n- 丰富的扩展性\nApache ShardingSphere 对于数据加密、分布式事务和影⼦库等⽅⾯都具备良好的扩展性。","date":"2022-11-01","author":"SphereEx ","excerpt":"通过架构重构，有效控制单表数据量，⼤幅缩减慢 SQL，下降将近 50%","metaDescription":null,"createdAt":"2022-11-03T06:03:40.665Z","updatedAt":"2022-11-16T09:39:37.310Z","publishedAt":"2022-11-03T06:03:44.426Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":{"id":423,"attributes":{"name":"WechatIMG1962.png","alternativeText":"WechatIMG1962.png","caption":"WechatIMG1962.png","width":1790,"height":758,"formats":{"thumbnail":{"name":"thumbnail_WechatIMG1962.png","hash":"thumbnail_Wechat_IMG_1962_95b9c6004e","ext":".png","mime":"image/png","width":245,"height":104,"size":41.69,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/thumbnail_Wechat_IMG_1962_95b9c6004e.png"},"large":{"name":"large_WechatIMG1962.png","hash":"large_Wechat_IMG_1962_95b9c6004e","ext":".png","mime":"image/png","width":1000,"height":423,"size":461.64,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/large_Wechat_IMG_1962_95b9c6004e.png"},"medium":{"name":"medium_WechatIMG1962.png","hash":"medium_Wechat_IMG_1962_95b9c6004e","ext":".png","mime":"image/png","width":750,"height":318,"size":283.18,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/medium_Wechat_IMG_1962_95b9c6004e.png"},"small":{"name":"small_WechatIMG1962.png","hash":"small_Wechat_IMG_1962_95b9c6004e","ext":".png","mime":"image/png","width":500,"height":212,"size":140.24,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/small_Wechat_IMG_1962_95b9c6004e.png"}},"hash":"Wechat_IMG_1962_95b9c6004e","ext":".png","mime":"image/png","size":966.17,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/Wechat_IMG_1962_95b9c6004e.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-11-03T06:01:48.589Z","updatedAt":"2022-11-03T06:01:48.589Z"}}},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":29,"attributes":{"feature":true,"title":"Apache ShardingSphere 在转转亿级交易系统落地实践","content":"Apache ShardingSphere在转转亿级交易系统落地实践\n## 一、背景与挑战\n这几年随着转转二手业务的快速发展，订单系统的基础性能问题也愈发严重，作为系统运转的基石，订单库压力不容小觑。 面临的问题：\n\n* 大促期间DB压力大，单库查询qps上万占用大量数据库资源，写性能大大降低；\n\n* 数据与日剧增，单库中包含非常多数据量过数亿的大表，占用空间接近服务器的容量上限；\n* 数据量太大，数据备份和恢复需要耗费很长时间，极端情况下丢失数据的风险越高。\n\n\n## 二、为什么选ShardingSphere\n迫于上述所说的DB压力问题，起初我们做了一些缓解措施，其中包括:\n\n- 优化大事务，缩小事务范围，甚至消除事务\n\n![图片](https://mmbiz.qpic.cn/mmbiz_png/dHUzltsJpQvfq4yuZUZnaFySYwp5ImicXKkqdaSqd8r1g1hw53d3ps0u0lPrKAY39IrXE9EOcVsevianzHNUuqZA/640?wx_fmt=png&wxfrom=5&wx_lazy=1&wx_co=1)]\n\n通过调整原有业务逻辑顺序将核心的生单步骤放置在最后 ，仅在订单主表保持事务，主表操作异常时其他订单相关的表允许有脏数据产生。\n\n- 订单数据添加缓存\n\n![[图片]](https://mmbiz.qpic.cn/mmbiz_png/dHUzltsJpQvfq4yuZUZnaFySYwp5ImicXTvwRIHyN6TyERnd9vHloiasooia9bmfU2KdzKeJKrvZhu1BEaAeuqjicA/640?wx_fmt=png&wxfrom=5&wx_lazy=1&wx_co=1)\n\n缓存这块最重要同时最麻烦的地方是保证数据的一致性，订单信息涉及结算抽佣等，数据的不实时或不一致会造成严重事故。   严格保证缓存数据的一致性，代码实现比较复杂同时会降低系统的并发，因此缓存方案实现这块我们做了一定的妥协：1. 允许数据缓存失败情况下请求直接查库;2. 给缓存key添加版本号，通过读最新版本号的数据确保数据的实时性。\n\n- 复杂查询走ES、主从分离、一些大表进行冷热数据分离等\n\n![[图片]](https://mmbiz.qpic.cn/mmbiz_png/dHUzltsJpQvfq4yuZUZnaFySYwp5ImicXzz3vmTHe65ujHNRIWQvibnCwkre6Rg3ywArCBpqVJdibW1PfE5IgRcGQ/640?wx_fmt=png&wxfrom=5&wx_lazy=1&wx_co=1)\n\n通过上述几个方面的优化DB压力虽有所下降，但面对大促等一些高并发场景时仍显得力不从心。为了从根本上解决订单库的性能问题，我们决定对订单库进行数据分片（分库分表）改造，使得未来3-5年内不需要担心订单容量问题。\n数据分片组件选型这块，我们从效率、稳定性、学习成本和时间多个方面对比，最终选择了ShardingSphere。\n[图片]\n\n## ShardingSphere优势如下:\n\n- 提供标准化的数据分片、分布式事务和数据库治理功能，可适用于如Java同构、异构语言、容器、云原生等各种多样化的应用场景；\n- 分片策略灵活，支持多种分片方式；\n- 集成方便、业务侵入程度低；\n- 文档丰富、社区活跃。ShardingShpere提出DB Plus的理念，采用可插拔的架构设计，模块相互独立，可以单独使用，亦可以灵活组合使用。目前ShardingSphere 由 JDBC、Proxy 和 Sidecar（规划中）这 3 款既能够独立部署，又支持混合部署配合使用的产品组成。 \n\n3款产品特性对比如下：\n\n[![图片]](https://mmbiz.qpic.cn/mmbiz_png/dHUzltsJpQvfq4yuZUZnaFySYwp5ImicXjenibENKwjBdT4jC8093bFZ9fWnhL8Jd4ZeNcrJ9a4l964QSqww6OfQ/640?wx_fmt=png&wxfrom=5&wx_lazy=1&wx_co=1)\n\n通过上图对比，结合订单高并发特性，本次数据分片中间件选择了ShardingSphere-JDBC。   ShardingSphere-JDBC定位为轻量级 Java 框架，在JDBC层提供的额外服务。它使用客户端直连数据库，以 jar 包形式提供服务，无需额外部署和依赖，可理解为增强版的 JDBC 驱动，完全兼容 JDBC 和各种 ORM 框架。\n\n![[图片]](https://mmbiz.qpic.cn/mmbiz_png/dHUzltsJpQvfq4yuZUZnaFySYwp5ImicXMuiaTH16tr615K1HoVGQuXKYbicKrcQHFbUaWicAjLbicibc6nicQuZxGkYg/640?wx_fmt=png&wxfrom=5&wx_lazy=1&wx_co=1)\n\n随着5.x版本的发布，ShardingSphere还提供了许多新特性：\n\n1. 全新 distsql 用于加载及展示 shardingsphere配置信息\n2. 支持跨不同数据库实例的分片 join sql 查询\n3. 增加数据网关能力，支持异构数据库存储\n4. 支持在线动态创建及修改用户权限\n5. 新增自动化探针模块\n\n## 三、项目落地中的关键点\n### 分片key的选择\n当前订单id的生成，采用了时间戳+用户标识码+机器码+递增序列的方式，其中用户标识码取自买家id的第9~16位，该码是用户id生成时的真随机位，很适合作为分片key。\n\n- 选择该用户标识码作为分片key有如下优势：\n  - 可以使分到各个库表的数据尽可能均匀；\n  - 无论是以订单id、还是以买家id作为查询条件，都可以快速定位到具体分片位置；\n  - 同一买家的数据能分布到相同的库表，方便买家维度的聚合查询。\n \n![[图片]](https://mmbiz.qpic.cn/mmbiz_png/dHUzltsJpQvfq4yuZUZnaFySYwp5ImicX3xGKDAYaxibeGG7tnBUVQwKXCVVwbUZziaQ0S2MQ7dMnWquIguXarlpg/640?wx_fmt=png&wxfrom=5&wx_lazy=1&wx_co=1)\n\n具体分片策略上：我们采用了16库16表，用户标识码取模分库，用户标识码高四位取模分表。\n\n### 新老库数据迁移\n\n  - 迁移必须是在线的，不考虑停机迁移，迁移的过程中还会有数据的写入；\n  - 数据应该是完整的，C端体验是无感知的，迁移之后需要保证新库和旧库的数据是严格一致的；\n  - 迁移过程中需要做到可以回滚，一旦迁移过程中出现问题，可以立即回滚到源库，不会对系统可用性造成影响。\n  数据迁移步骤如下：双写->迁移历史数据->校验->老库下线。\n## 四、效果&收益\n\n- 解决了单库容量上限问题;\n- 数据分片后单库表的数据量大大减少，单表数据量由原来的近亿降为百万级别，总体性能大大提高;\n- 降低了因单库、单表数据过大极端情况数据丢失风险，减轻运维压力。\n以下是两次大促期间，下单服务接口调用量与接口耗时对比。\n\n改造前：\n\n![[图片]](https://mmbiz.qpic.cn/mmbiz_png/dHUzltsJpQvfq4yuZUZnaFySYwp5ImicXflshq7lNHV5g3mZPRz9tRcbzXODDaP9kCkU6v6KUrxQkFUJvFgINcQ/640?wx_fmt=png&wxfrom=5&wx_lazy=1&wx_co=1)\n\n改造后：\n\n![[图片]](https://mmbiz.qpic.cn/mmbiz_png/dHUzltsJpQvfq4yuZUZnaFySYwp5ImicXzUDnSOkNjMdRh9jDian4JciaU6dXXkWZv84pUHicFZx6sSsIm621pevlg/640?wx_fmt=png&wxfrom=5&wx_lazy=1&wx_co=1)\n\n## 五、总结感悟\n  任何公司的“分库分表项目”说白了，都不是一个考验点子思路的常见项目，更多的反而是对一个人、一个团队干活的细致程度、上下游部门的沟通协作、工程化的操作实施以及抗压能力的综合考验。同时业务的不断发展，又促使系统数据架构需要跟着不断升级，ShardingSphere以其良好的架构设计、高度灵活、可插拔和可扩展的能力简化了数据分片的开发难度，使研发团队只需关注业务本身，进而实现了数据架构的灵活扩展。","date":"2022-11-01","author":"张强，转转交易中台研发工程师","excerpt":"业务的不断发展，促使系统数据架构的不断升级","metaDescription":null,"createdAt":"2022-11-03T08:13:11.607Z","updatedAt":"2023-03-21T07:31:16.169Z","publishedAt":"2022-11-03T08:13:14.490Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":{"id":354,"attributes":{"name":"zhuanzhuan logo.png","alternativeText":"zhuanzhuan logo.png","caption":"zhuanzhuan logo.png","width":130,"height":130,"formats":null,"hash":"zhuanzhuan_logo_fd59d35610","ext":".png","mime":"image/png","size":1.85,"url":"https://official-media.sphere-ex.com/zhuanzhuan_logo_fd59d35610.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:45:59.332Z","updatedAt":"2022-09-26T08:45:59.332Z"}}},"metaLogo":{"data":{"id":354,"attributes":{"name":"zhuanzhuan logo.png","alternativeText":"zhuanzhuan logo.png","caption":"zhuanzhuan logo.png","width":130,"height":130,"formats":null,"hash":"zhuanzhuan_logo_fd59d35610","ext":".png","mime":"image/png","size":1.85,"url":"https://official-media.sphere-ex.com/zhuanzhuan_logo_fd59d35610.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:45:59.332Z","updatedAt":"2022-09-26T08:45:59.332Z"}}},"localizations":{"data":[]}}},{"id":30,"attributes":{"feature":true,"title":"基于 ShardingSphere 的得物数据库中间件平台“彩虹桥”演进之路","content":"## 前言\n随着得物 App 用户开始快速增长，业务线日趋丰富，也对底层数据库带来了较大的压力。各个业务线对于数据分片、读写分离、影子库路由等等的需求成为了刚需，所以需要一个统一的中间件来支撑这些需求，得物“彩虹桥”应运而生。\n\n在北欧神话中，彩虹桥是连结阿斯加德（ Asgard ）【1】和 米德加尔特（中庭/ Midgard ）的巨大彩虹桥。我们可以把它当作是“九界之间”的连接通道，也是进入阿斯加德的唯一稳定入口。而得物的彩虹桥是连接服务与数据库之间的数据库中间层处理中间件，可以说得物的每一笔订单都与它息息相关。\n\n## 1. 技术选型\n\n前期我们调研了 Mycat、ShardingSphere、kingshard、Atlas 等开源中间件，综合了适用性、优缺点、产品口碑、社区活跃度、实战案例、扩展性等多个方面，最终我们选择了 ShardingSphere 。准备在 ShardingSphere 的基础上进行二次开发、定制一套适合得物内部环境的数据库中间件。\n\nApache ShardingSphere 是一款开源分布式数据库生态项目，由 JDBC、Proxy 和 Sidecar（规划中） 3 款产品组成。其核心采用可插拔架构，通过组件扩展功能。对上以数据库协议及 SQL 方式提供诸多增强功能，包括数据分片、访问路由、数据安全等；对下原生支持 MySQL、PostgreSQL、SQL Server、Oracle 等多种数据存储引擎。ShardingSphere 已于2020年4月16日成为 Apache 软件基金会的顶级项目，并且在全球多个国家都有团队在使用。\n\n目前我们主要是 ShardingSphere的Proxy 模式提供服务，后续将会在 JDBC&Proxy 混合架构继续探索。\n\n## 2.彩虹桥目前的能力\n\n![](https://oscimg.oschina.net/oscnet/up-8589dc26cf4195958a56f1878375ad2ab62.png)\n\n其中白色模块为现阶段以及具备的能力，绿色模块为规划&正在做的功能。下面介绍一下几个重点功能。注意，以下功能都是基于 Proxy 模式。\n\n### 2.1 数据分片\n\n数据分片指按照某个维度将存放在单一数据库中的数据分散地存放至多个数据库或表中以达到提升性能瓶颈以及可用性的效果。数据分片的有效手段是对关系型数据库进行分库和分表。分库和分表均可以有效地避免由数据量超过可承受阈值而产生的查询瓶颈。除此之外，分库还能够用于有效地分散对数据库单点的访问量；分表虽然无法缓解数据库压力，但却能够提供尽量将分布式事务转化为本地事务的可能，一旦涉及到跨库的更新操作，分布式事务往往会使问题变得复杂。使用多主多从的分片方式，可以有效地避免数据单点，从而提升数据架构的可用性。\n\n通过分库和分表进行数据的拆分来使得各个表的数据量保持在阈值以下，以及对流量进行疏导应对高访问量，是应对高并发和海量数据系统的有效手段。数据分片的拆分方式又分为垂直分片和水平分片。\n\n按照业务拆分的方式称为垂直分片，又称为纵向拆分，它的核心理念是专库专用。彩虹桥主要提供的分片能力是水平分片，水平分片又称为横向拆分。相对于垂直分片，它不再将数据根据业务逻辑分类，而是通过某个字段（或某几个字段），根据某种规则将数据分散至多个库或表中，每个分片仅包含数据的一部分。例如：根据主键分片，偶数主键的记录放入 0 库（或表），奇数主键的记录放入 1 库（或表），如下图所示。\n\n![](https://oscimg.oschina.net/oscnet/up-03995f1fc41e54091d476d67a3a3146a382.png)\n\n水平分片从理论上突破了单机数据量处理的瓶颈，并且扩展相对自由，是数据分片的标准解决方案。\n\n当然实际使用场景的分片规则是非常复杂的，我们提供一些内置算法比如取模、HASH 取模、自动时间段分片算法、Inline 表达式等。当内置算法无法满足要求时，还可以基于 groovy 来定制专属的分片逻辑。\n\n![](https://oscimg.oschina.net/oscnet/up-6d53404d8c0b8a4373c971f612169a5bd11.png)\n\n### 2.2 读写分离\n\n面对日益增加的系统访问量，数据库的吞吐量面临着巨大瓶颈。对于同一时刻有大量并发读操作和较少写操作类型的应用系统来说，将数据库拆分为主库和从库，主库负责处理事务性的增删改操作，从库负责处理查询操作，能够有效的避免由数据更新导致的行锁，使得整个系统的查询性能得到极大的改善。通过一主多从的配置方式，可以将查询请求均匀地分散到多个数据副本，能够进一步地提升系统的处理能力。\n\n与将数据根据分片键打散至各个数据节点的水平分片不同，读写分离则是根据 SQL 语义的分析，将读操作和写操作分别路由至主库与从库。\n\n![](https://oscimg.oschina.net/oscnet/up-3afd2cc77cd8369c2bb82b177f916342753.png)\n\n这里配置的方式比较简单，给目标主库绑定一个或多个从库、设置对应的负载均衡算法即可。\n\n![](https://oscimg.oschina.net/oscnet/up-5b17dae8d16b16886913af6bde92c238c27.png)\n\n这里的实现方式就是通过 SQL 解析，把查询语句路由到对应的从库即可，但是在一些对主从同步延迟比较敏感的场景，可能需要强制走主库，这里我们也提供一个 API（原理就是 SQL Hint），让上游可以指定某些模块读强制走主，还有相关全局配置可以让事务内所有读请求全部走主。\n\n### 2.3 影子库压测\n\n在基于微服务的分布式应用架构下，业务需要多个服务是通过一系列的服务、中间件的调用来完成，所以单个服务的压力测试已无法代表真实场景。在测试环境中，如果重新搭建一整套与生产环境类似的压测环境，成本过高，并且往往无法模拟线上环境的复杂度以及流量。因此，业内通常选择全链路压测的方式，即在生产环境进行压测，这样所获得的测试结果能够准确地反应系统真实容量和性能水平。\n\n全链路压测是一项复杂而庞大的工作。需要各个微服务、中间件之间配合与调整，以应对不同流量以及压测标识的透传。通常会搭建一整套压测平台以适用不同测试计划。在数据库层面需要做好数据隔离，为了保证生产数据的可靠性与完整性，需要将压测产生的数据路由到压测环境数据库，防止压测数据对生产数据库中真实数据造成污染。这就要求业务应用在执行 SQL 前，能够根据透传的压测标识，做好数据分类，将相应的 SQL 路由到与之对应的数据源。\n\n这里配置的方式类似读写分离，也是给目标主库绑定一个影子库，当 SQL 携带了影子标就会被路由到影子库。\n\n![](https://oscimg.oschina.net/oscnet/up-043144bd50a5daee6ff5781f3cf5236f6f5.png)\n\n### 2.4 限流&熔断\n\n当 DB 的压力超过自身水位线时，会导致 DB 发生故障。当我们预估出某个维度的水位线后，可以配置对于的限流规则，拒绝掉超过本身水位线以外的请求来保护 DB。让系统尽可能跑在最大吞吐量的同时保证系统整体的稳定性。维度方面我们支持 DB、Table、SQL 以及 DML 类型。\n\n![](https://oscimg.oschina.net/oscnet/up-0f50d0618386e4678c772a164ca35091bf3.png)\n\n彩虹桥下面连接了上百个 RDS 实例，每个 RDS 实例都有可能出现各种故障，当单个实例出现故障会影响到整个逻辑库，会迅速造成阻塞诱发雪崩效应。所以我们需要一种快速失败的机制来防止雪崩，目前我们是支持 DB 实例级别的熔断，基于获取连接& SQL 执行 2 种行为，以及执行时间跟失败比例来实现熔断，以达到在 DB 故障时快速失败的效果。\n\n![](https://oscimg.oschina.net/oscnet/up-916a3c6015f00af9c677499da18fc67558e.png)\n\n### 2.5 流量纠偏\n\n在双活架构下，彩虹桥作为数据库的代理层，可以保证双活架构下流量切换过程的流量做兜底拦截，保证数据一致性。原理就是基于 SQL Hint 携带的 userId 与机房规则做匹配，拦截不属于当前机房的流量。\n\n## 3. 基于 ShardingSphere 我们做了哪些改造\n\n虽然 ShardingSphere Proxy 本身其实已经足够强大，但是针对得物内部环境，还是存在一些缺陷和功能缺失，主要分为以下几点：\n\n* 易用性\n\n    * 分片、读写分离等规则配置文件过于复杂，对于业务开发不够友好\n\n    * 规则动态变更完全依赖配置中心，缺失完善的变更流程\n\n    * 连接池治理能力不完善\n\n    * Hint 方式不够友好，需要业务写 RAL 语句\n\n    * 自定义分片算法需要发布\n\n    * SQL 兼容性\n\n* 稳定性\n\n    * 多集群治理功能缺失\n    \n    * 逻辑库之间的隔离性缺失\n     \n    * 限流熔断组件缺失\n     \n    * 数据源、规则动态变更有损\n     \n    * 双活架构下流量纠偏功能的缺失\n     \n    * 发布有损\n\n* 可观测性\n \n    * SQL Trace 功能不完善\n     \n    * 监控指标不够全面\n     \n    * SQL 洞察能力缺失\n\n* 性能\n\n    * 由于多了一次网络转发，单条 SQL 的 RT 比直连会上浮 2~3ms\n\n为了解决以上问题，我们做了以下改造与优化。\n\n### 3.1 易用性提升\n\n* 针对数据源、规则的配置、变更、审计等一系列操作，集成到管控台进行统一操作。通过图形化的方式降低了分片、读写分离等规则配置文件的复杂度，并且加上一系列校验来规避了一部分因为配置文件错误导致的低级错误。其次加上了审计功能，保障了配置动态变更的安全性和可控性。\n\n* 管控台新增了连接池治理，基于 RDS 连接数、Proxy 节点数、挂载数据源数量等因素自动计算出一个安全合理的连接池大小。\n\n* 新增 Client，针对 Hint 做一系列适配，比如影子标传递、双活架构用户 id 传递、trace 传递、强制路由等等。使用方只需要调用 Client 中的 API，即可在发出 SQL 阶段自动改写成 Proxy 可以识别的 Hint 增强语句。\n\n* 管控台新增了集群治理功能：由于我们部署了多套 Proxy 集群，为了最大程度的保障故障时的爆炸半径，我们按照业务域对 Proxy 集群进行了划分，尽量保证统一业务域下面的逻辑库流量进入同一套集群。\n\n* 新增了 Groovy 来支持自定义分片算法，在后台配置好分片逻辑后审核通过即可生效，无需 Proxy 发版\n\n### 3.2 稳定性提升\n\n#### 3.2.1 Proxy 多集群治理\n\n（1）背景\n\n随着 Proxy 承载的业务域越来越多，如果所有的流量都通过负载均衡路由到Proxy 节点，当一个库出现问题时，可能会导致整个集群瘫痪，爆炸范围不可控。而且由于 DB 的连接数资源有限，这就导致 Proxy 的节点无法大规模横向扩展。\n\n（2）解决方案\n\n为了不同业务域之间的隔离性，我们部署了多套 Proxy 集群，并且通过管控台维护各个逻辑库与集群之间的关系。保证同一业务域下面的逻辑库流量进入同一套集群。并且在某个集群发生故障时，可把故障集群中的逻辑库迅速、无损的动态切换至备用集群，尽可能减少 Proxy 本身故障给业务带来的损失。\n\n而针对连接数治理方面，Proxy 在初始化连接池的时候会判断一下当前逻辑库是否在当前集群，如果不在把最小连接数配置设置成最小，如果在则按照正常配置加载，并在集群切换前后做好目标集群的预热以及原集群的回收。这样可以极大程度上缓解 DB 连接池资源对 Proxy 节点的横向扩展。\n\n（3）实现原理\n\n上游应用引入 Rainbow（自研连接池）后，在连接池初始化之前会根据逻辑库读取当前库所在集群，并动态把 Proxy 的域名替换成其所在集群的域名。同时还会新增对集群配置的监听，这样在管控台切换集群操作后，Rainbow 会根据切换后的集群域名创建一个新的连接池，然后替换掉老的连接池，老的连接池也会进行延时优雅关闭，整个过程对上游应用无感知。\n\n（4）架构图\n\n![](https://oscimg.oschina.net/oscnet/up-79cf73883082b7b4f2a710c21ddee45ac22.png)\n\n#### 3.2.2 Proxy 工作线程池隔离\n\n（1）背景\n\n开源版本的 Proxy，所有逻辑库共用一个线程池，当单个逻辑库发生阻塞的情况，会耗尽线程池资源，导致其他逻辑库也跟着受影响。\n\n（2）解决方案\n\n我们这里采用了基于线程池的隔离方案，引入了独占&共享线程池的概念，优先使用逻辑库独占线程池，当独占线程池出现排队情况再使用共享线程池，共享线程池达到一定负载后会在一个时间窗口内强制路由到独占线程池，在保障隔离性的前提下，最大化利用线程资源。\n\n#### 3.2.3 流控与熔断\n\n（1）背景\n\n开源版本的 Proxy缺少对库、表、SQL 等维度的流控，当短时间内爆发超过系统水位的流量时，很可能就会击垮 DB 导致线上故障，而且缺少针对 DB 快速失败的机制，当 DB 发生故障（比如 CPU 100%）无法快速失败的情况下，会迅速造成阻塞诱发雪崩效应。\n\n（2）解决方案\n\n新增了各个维度（DB、table、SQL、语句类型）的限流，各个库可以根据预估的水位线以及业务需要配置一个合理的阈值，最大程度保护 DB 。同时我们也引入 DB 实例的熔断策略，在某个实例出现故障时可快速失败。在分库场景下，可最大程度减少对其他分片的影响，进一步缩小了故障的爆炸半径，提升彩虹桥整体的稳定性。\n\n（3）实现原理\n\n流控跟熔断都是基于 sentinel 来实现的，在管控台配置对应的规则即可。\n\n#### 3.2.4 无损发布\n\n（1）背景\n\n在前期，Proxy 每次发布或者重启的时候，都会收到上游应用 SQL 执行失败的一些报警，主要的原因是因为上游与 Proxy 之间是长连接，这时如果有连接正在执行 SQL，那么就会被强制断开导致 SQL 执行失败，一定程度上对上游的业务造成损失。\n\n（2）解决方案\n\n发布系统配合自研连接池 Rainbow，在 Proxy 节点发布或重启之前，会通知 Rainbow 连接池优雅关闭应用于需要重启&发布的 Proxy 节点之间的连接，在 Proxy 流量跌 0 后再执行重启&发布。\n\n### 3.3 可观测性\n#### 3.3.1 运行时指标\n\n（1）背景\n\n开源版本对于 Proxy 运行时的监控指标很少，导致无法观测到 Proxy 上面每个库运行的详细状态。\n\n（2）解决方案\n\n在 Proxy 各个执行阶段加了埋点，新增了以下指标、并绘制了对应的监控大盘。\n\n* 库&表级别的 QPS、RT、error、慢 SQL 指标\n \n* Proxy-DB 连接池的各项连接数指标\n \n* 流控熔断指标\n \n* 线程池活跃线程数、队列大小指标\n\n（3）效果图\n\n![](https://oscimg.oschina.net/oscnet/up-c7bb9bdd9ab8438333c1c801bbbf485c02e.png)\n\n![](https://oscimg.oschina.net/oscnet/up-63f959dba22c7f0ffb5646aa08da29708a7.png)\n\n![](https://oscimg.oschina.net/oscnet/up-b7d5d5e4c1595ace1819fabe3a29eb9bb2f.png)\n\n#### 3.3.2 全链路追踪\n\n（1）背景\n\n开源版本的 trace 只支持 Proxy 内部执行阶段的链路追踪，无法和上游串联，导致排障效率低下。\n\n（2）解决方案\n\n主要通过 RAL、SQL 注释 2 种方式传递 trace 信息，实现了上游到 Proxy 的全链路追踪。\n\n（3）实现原理\n\n我们首先想到的方案就是通过 SQL 注释方式传递 trace 信息，但是在真正投产之后却发现了一些问题，在上游使用了 prepare 模式（useServerPrepStmts=true&cachePrepStmts=true）时，Proxy 会缓存statmentId 和 SQL ，而 trace 每次都不一样，这样会导致存储缓存无限增长最终导致 OOM 。\n\n![](https://oscimg.oschina.net/oscnet/up-1fad2c4ac07b66b974efdf6f5886ff3abbd.png)\n\n所以通过 SQL 注释方式传递 trace 信息只适用于非 prepare 场景。于是我们又新增了一种方案就是在每次 SQL 执行之前发送一条 RAL 语句来传递 trace 信息，并在 Proxy 中缓存 channelId 与 trace 信息的对应关系，由于单个channel所有的SQL都是串行执行，加上 channelId 数量可控，不会无限膨胀。但是这种方案相对于每次 SQ L执行之前都有一次 RAL 语句的执行，对性能的影响还是比较大的。从监控上看下来每次 SQL 执行的 RT 会上浮 2-3ms（网络传输），对于一些链路较长的接口来说还是挺致命的。\n\n（4）总结\n\n综合下来 2 种方案其实都有比较明显的缺点，针对这个问题之前 ShardingSphere 的小伙伴来过我们得物进行过一次深度沟通，亮哥给出了一个比较可行的方案，就是通过虚拟列传输trace信息，但是需要上游对 SQL 进行改写，这可能会增加上游应用的负担，目前这块我们还没有开始做。\n\n#### 3.3.3 SQL 洞察\n\n（1）背景\n\n目前 Proxy 虽然有打印逻辑 SQL 和物理 SQL 的日志，但是由于生产请求量较大，开启日志会对 IO 有较大挑战。所以目前我们生产环境还是关闭的状态。而且这个日志也无法与上游串联，对于排障的帮助也是比较有限。\n\n（2）解决方案\n\n目前也只有个大概的思路，还没有完善的方案，要达到的效果就是收集所有 Proxy 执行的逻辑 SQL、物理 SQL 以及上游信息，包括 JDBCDatabaseCommunicationEngine 以外的一些 SQL（比如 TCL 等等），并通过管控台实时查询。最终的效果类似阿里云 RDS 的收费服务-SQL 洞察\n\n### 3.4 bug 修复\n\n由于历史原因，我们是基于 Apache ShardingSphere 5.0.0-alpha 上做的二次开发，在实际使用的过程中遇到了很多 bug，大部分都给官方提了 issue，并且在彩虹桥版本上做了修复，当然 ShardingSphere 社区的小伙伴也给与了很多帮助和修复的思路。\n\n### 3.5 JDBC&Proxy 混合架构\n\n（1）背景\n\n上面 4 项内容大多数针对 Proxy 模块或上游连接池模块的改造，但是还是有一些问题是 Proxy 模式暂时无法解决的，比如前面提到的 SQL 兼容性和性能问题。还有就是如果整个 Proxy 集群全部宕机情况下我们没有一个兜底机制。所以 Proxy 模式并不适用于所有场景。在 Apache ShardingSphere 的官方文档可以看到这样一段内容：\n\n![](https://oscimg.oschina.net/oscnet/up-2e0d1f8836d71db1cea63aa5b1cda5f1057.png)\n\n于是我们准备在 JDBC&Proxy 混合架构上做了进一步探索。\n\n（2）解决方案\n\n在管控台实现对逻辑库的模式配置，并通过自研连接池 Rainbow 感知并根据不同模式启动不同类型的数据源。并且可以在后台切换模式后无损动态调整，这样对于使用者来说是完全无感知的。对 SQL 性能、兼容性要求比较高的应用可以调整为 JDBC 模式。同时当 Proxy 所有集群瘫痪时也有个兜底的方案，不至于全站崩溃。\n\n（3）实现原理\n\nRainbow 连接池启动的时候会查询当前逻辑库对应的模式，如果是 Proxy 模式则直接连接 Proxy 来启动连接池，如果是 JBDC 则根据该逻辑库的数据源配置以及分片&读写分离&影子库等等规则来加载 JDBC 模式的数据源，对应的 DataSource 为 GovernanceShardingSphereDataSource。并且会监听这个模式配置，当模式发生变化后会动态无损替换当前连接池。具体的无损替换方案与前面提到的集群切换类似。同时需要解决的还有监控问题和连接池资源的管理，Proxy 切换至 JDBC 模式后指标的暴露由 Proxy 节点变成了上游节点，对应的大盘也需要做对应的融合，连接池治理这块也使用了新的计算模式做了对应的适配。\n\n## 4. 目前的困惑\n\n由于历史原因，我们是基于 Apache ShardingSphere 5.0.0-alpha 上做的二次开发，目前社区最新的版本是 5.1.2-release，从 5.0.0-alpha ~ 5.1.2-release 做了大量的优化跟 bug 修复，但是我们没有很好的办法将社区的代码合并到我们的内部代码，因为无法确定社区开源版本更改的内容，会不会对现有业务产生影响，短时间内也无法享受社区带来的红利。同时我们也在寻找一种方式将我们做的一些优化后续合并到社区中，也算是一种反哺社区。为中国开源做一份贡献~","date":"2022-11-01","author":"陈浩","excerpt":"得物的彩虹桥是连接服务与数据库之间的数据库中间层处理中间件，可以说得物的每一笔订单都与它息息相关。","metaDescription":null,"createdAt":"2022-11-03T09:00:48.784Z","updatedAt":"2022-11-03T09:00:51.175Z","publishedAt":"2022-11-03T09:00:51.172Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":31,"attributes":{"feature":true,"title":"ShardingSphere 在数十个联通政务热线场景中的应用：稳定、高效、可复制","content":"联通数科整合中国联通“云、大、物、智、链、安”（云计算、大数据、物联网、人工智能、区块链、安全）等能力，面向智慧城市、数字政府、工业互联网、生态环保、文化旅游等领域，联通数科已积累大量行业标杆案例，成功为客户提供更加丰富、专业的产品和服务。\n\n## 一、项目背景\n\n随着“互联网+政务服务”政策的持续发力，越来越多的地方政府开始着手打造“互联网+”环境下的政务服务平台。\n\n近年来，联通数科先后助力数十个政务热线平台上线，以提升政务服务能力，部分项目如下图。\n\n![up-20ec9e57d65c64dcf55a4277105b5ff0488.webp](https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/up_20ec9e57d65c64dcf55a4277105b5ff0488_6143b4132c.webp)\n\n政务热线业务有并发高、数据量大的特点，我们每拨打一次政务热线，即产生一条工单记录。相比以往，疫情期间的热线业务量更是翻了数倍。\n\n**在政务热线的工单模块中，为满足大规模、海量数据以及高稳定的业务需求，联通数科采用 ShardingSphere 对工单信息进行分片存储，在政民互动的桥梁中构建坚固的桥墩**。\n\n## 二、业务挑战\n\n政务热线是政府与企业、群众互动的主要渠道，以单一号码面对公众，提供 7 * 24 服务。除受理工单外，还包括了话务、知识库、语音对答及智能坐席等业务。\n\n在系统规划阶段，数据库层作为业务系统的基石，技术选型过程更为谨慎。系统的稳定是硬性指标，其次是性能，性能直接影响热线业务的服务能力，同时还要具备易于维护和管理的特点，便于系统升级及后台维护。\n\n在数据库架构选型中，政务热线系统对架构要求如下：\n\n-   成熟稳定\n    \n-   高性能\n    \n-   易维护\n    \n-   业务代码低耦合\n    \n\n## 三、选择 ShardingSphere 的理由\n\n**经过技术团队多轮调研测试，包括稳定性、产品功能、接入方式及性能等，最终选择“ShardingSphere + MySQL”分布式数据库解决方案**。\n\n-   理念先进\n    \n\n随业务应用场景多元化发展，数据库已呈碎片化趋势，「**Database Plus**」旨在数据库上层建立标准和生态，为数据库提供增量能力，可防止数据库绑定、实现自主可控，突破原有数据库功能局限，提升数据基础设施整体功能上限。\n\n-   成熟稳定/案例多\n    \n\nShardingSphere 于 2016 年开源，经过多年的研发迭代，产品历经多个头部互联网大型场景打磨，成熟稳定、值得信赖。\n\n-   功能全面\n    \n\nShardingSphere 除了具备 Sharding 能力外，数据加解密和影子库的功能也是作为联通技术团队评估的指标，通过一套技术栈既可解决安全和压测的场景需求。ShardingSphere 为政务热线系统建设及架构扩展，提供了更多的能力支持。\n\n-   高兼容/接入方便\n    \n\nShardingSphere 与 MySQL 协议兼容，语法支持度也非常高，在开发过程中几乎不用考虑 SQL 的支持问题，接入非常方便。\n\n-   高性能\n    \n\n政务热线忙时有数千坐席在线，工单记录以千万记录为起始单位，对性能有明确的要求。ShardingSphere-JDBC 定位为轻量级的 Java 框架，压测结果数据理想，可满足政务系统服务能力要求。\n\n-   易运维\n    \n\n在 JDBC 良好的性能基础之上，ShardingSphere 还提供了对运维人员友好的 Proxy，使用常用的客户端即可直接访问，上手维护较容易。\n\n除了以上五个维度的权衡，选型中也参考了其他数据库中间件产品，以下表格是 ShardingSphere 和 Mycat 的比对。\n\n|   | Apache ShardingSphere | Mycat |\n| --- | --- | --- |\n| 开源协议 | Apache-2.0 | GPL-2.0/GPL-3.0 |\n| 接入端 | 客户端/代理端 | 代理端 |\n| 数据库支持 | 客户端：任意  \n代理端：MySQL/PG/openGauss | MySQL、Oracle、DB2、SQL Server、PostgreSQL 等 |\n| 性能 | 客户端：损耗低  \n代理端：损耗高 | 损耗高 |\n| 读写分离 | 有 | 有 |\n| 数据加密 | 有 | 无 |\n| 全链路压测 | 有 | 无 |\n| 社区成熟度 | Apache 软件基金会  \n信通院可信认证  \n社区活跃度高 | 个人/三方社区  \n社区活跃度一般 |\n| 商业公司支持 | 有 | 无 |\n\n## 四、解决方案\n\n由于“ShardingSphere+MySQL ”具备稳定、易用及性能极致等特点，该方案在联通数科的多个政务热线项目中复制使用。\n\n-   部署方式\n    \n\nShardingSphere-JDBC 和 ShardingSphere-Proxy 混合使用的方式，兼顾了系统性能和可维护性。\n\n-   分片键/分片算法\n    \n\n政务热线的工单表相对较大，且使用逻辑简单，因此在分片设计上，选取工单 ID 作为分片键，通过哈希算法分布数据。\n\n-   迁移\n    \n\n新建项目不涉及历史数据，因此无需考虑数据迁移环节，上线即直接使用。\n\n-   实施过程\n    \n\n回忆数个已上线的项目，由于技术方案评估较全面，且多次复制方案，实施过程并没有遇到“坑”。\n\n-   系统架构\n    \n\n政务热线业务模块采用了微服务架构，节点在十几到几十不等，以具体省市规模决定。在数据库层，使用物理机部署 3 节点 MHA，每个政务热线系统一般需要 4 套 MHA 来支撑业务。\n\n![up-6eb6faa996c81e6de688a2dd9e204f730e6.webp](https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/up_6eb6faa996c81e6de688a2dd9e204f730e6_e867e826e4.webp)\n\n## 五、用户收益\n\n-   业务角度\n    \n    -   支持客户端模式，最大化保证了政务热线**对性能的要求**；\n        \n    -   **系统稳定**，为业务连续性带来有力的支撑。\n        \n-   研发角度\n    \n    -   **兼容良好**，兼容主流数据库 SQL 语法，对接即可使用；\n        \n    -   **功能全面**，读写分离、数据加密及全链路压测方案，系统扩展性强。\n        \n-   运维角度\n    \n    -   支持代理端模式，为运维人员提供了**良好的维护入口**。\n        \n-   技术选型角度\n    \n    -   **防止数据库绑定**，为未来升级提供了足够的灵活度支撑。\n        \n\n## 六、总结\n\n通过联通数科政务热线的多个案例，验证了 ShardingSphere 在政务场景的能力，也进一步说明了 ShardingSphere 无行业限制的特点。\n\n目前我国数字政府建设已进入全面提速阶段，联通数科和 ShardingSphere 通力合作，希望助力更多政务热线项目落地，打造更多高效、便捷和安全的政务热线。\n\n","date":"2022-11-01","author":"SphereEx 伙伴","excerpt":"通过联通数科政务热线的多个案例，验证了 ShardingSphere 在政务场景的能力，也进一步说明了 ShardingSphere 无行业限制的特点。","metaDescription":null,"createdAt":"2022-11-03T09:06:14.351Z","updatedAt":"2023-08-24T03:32:25.633Z","publishedAt":"2022-11-03T09:06:16.445Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":32,"attributes":{"feature":true,"title":" 数据为王，携程国际火车票的 ShardingSphere 之路","content":"以下文章来源于携程技术，作者瑞华\n\n**作者简介**\n\n瑞华，携程高级后端开发工程师，关注系统架构、分库分表、微服务、高可用等。\n\n## **一、前言**\n\n随着国际火车票业务的高速发展，订单量快速增长，单数据库瓶颈层面的问题逐渐显露，常规的数据库优化已无法达到期望的效果。同时，原先的底层数据库设计，也存在一些历史遗留问题，比如存在部分无用字段、表通过自增主键关联和各个应用直连数据库等问题。\n\n为此，经过讨论后，我们决定对订单库进行分库分表，同时对订单表进行重构，进而从根本上解决这些问题。\n\n## **二、问题挑战**\n\n目标确定后，实践起来可不轻松，出现了很多的问题和挑战。这里列举一些典型问题，大致可以分为两大类：分库分表通用问题、具体业务关联问题。\n\n**分库分表通用问题**\n\n-   如何切分，垂直分还是水平分？分片的键，如何选取？\n    \n-   如何根据键值路由到对应库、对应表？\n    \n-   采用什么中间件，代理方式还是中间件的方式？\n    \n-   跨库操作等问题，如跨库事务和跨库关联？\n    \n-   数据扩容问题，后续如何进行扩容？\n    \n\n**具体业务关联问题**\n\n-   各个应用直连数据如何解决？\n    \n-   如何进行平滑过渡？\n    \n-   历史数据如何恰当迁移？\n    \n\n## **三、方案选型**\n\n### **3.1 如何切分**\n\n切分方式，一般分为垂直分库、垂直分表、水平分库和水平分表四种，如何选择，一般是根据自己的业务需求决定。\n\n我们的目标是要从根本上解决数据量大、单机性能问题等问题，垂直方式并不能满足需求，所以我们选取了水平分库+水平分表的切分方式。\n\n### **3.2 分片键选取**\n\n一般是根据自己的实际业务，来选择字段来作为分片的键，同时可以结合考虑数据的热点问题 、分布问题。比如订单系统，不能根据国家字段进行分片，否则可能会出现某些国家很多的订单记录，某些国家几乎没有订单记录，进而数据分布不均。相对正确的方式，比如订单类系统，可以选择订单 ID；会员系统，可以选择会员 ID。\n\n### **3.3 如何路由**\n\n选定了分片的键之后，接下来需要探讨的问题，就是如何路由到具体的数据库和具体的表。以分片键路由到具体某一个数据库为例，常见的路由方式如下：\n\n#### **映射路由**\n\n映射路由，即新增一个库，新建一个路由映射表，存储分片键值和对应的库之间的映射关系。比如，键值为 1001，映射到 db01 这个数据库，如下图所示：\n\n![](https://oscimg.oschina.net/oscnet/up-cfd0ac43199f735ff69c5aa49842adaf1e4.png)\n\n映射方式，优点是映射方式可任意调整，扩容简单，但是存在一个比较严重的不足，就是映射库中的映射表的数据量异常巨大。我们本来的目标是要实现分库分表的功能，可是现在，映射库映射表相当于回到了分库分表之前的状态。所以，我们在实践中，没有采取这种方式。\n\n#### **分组路由**\n\n分组路由，即对分片的键值，进行分组，每组对应到一个具体的数据库。比如，键值为 1000 到 2000，则存储到 db01 这个数据库，如下图所示：\n\n![](https://oscimg.oschina.net/oscnet/up-e9961db378f4c5995994e452fe60a0ed897.png)\n\n分组方式，优点是扩容简单，实现简单，但是也存在一个比较严重的不足，是数据分布热点问题，比如在某一个时间内，分片键值为 2001，则在将来一段时间内，所有的数据流量，全部打到某一个库（db02）。这个问题，在互联网环境下，也比较严重，比如在一些促销活动中，订单量会有一个明显的飙升，这时候各个数据库不能达到分摊流量的效果，只有一个库在接收流量，会回到分库分表之前的状态。所以，我们也没有采取这种方式。\n\n#### **哈希路由**\n\n哈希路由，即对分片的键值，进行哈希，然后根据哈希结果，对应到一个具体的数据库。比如，键值为 1000，对其取哈希的结果为 01，则存储到 db01 这个数据库，如下图所示：\n\n![](https://oscimg.oschina.net/oscnet/up-d7875cca3405c6e251421a2a18f6f7be169.png)\n\n哈希方式，优点是分布均匀，无热点问题，但是反过来，数据扩容比较麻烦。因为在扩容过程中，需要调整哈希函数，随之带出一个数据迁移问题。互联网环境下，迁移过程中往往不能进行停服，所以就需要类似多库双写等方式进行过渡，比较麻烦。所以，在实践中也没有采取这种方式。\n\n#### **分组哈希路由**\n\n分组哈希路由，即对分片的键值，先进行分组，后再进行哈希。如下图所示：\n\n![](https://oscimg.oschina.net/oscnet/up-038c74382d3b91ed7a05b6c49c5435508b8.png)\n\n在实践中，我们结合了前面的几种方式，借鉴了他们的优点不足，而采用了此种方式。因为分组方式，能很方便的进行扩容，解决了数据扩容问题；哈希方式，能解决分布相对均匀，无单点数据库热点问题。\n\n### **3.4 技术中间件**\n\n分库分表的中间件选取，在行业内的方案还是比较多的，公司也有自己的实现。根据实现方式的不同，可以分为代理和非代理方式，下面列举了一些业界常见的中间件，如下表（截至于 2021-04-08）：\n\n![](https://oscimg.oschina.net/oscnet/up-0d958dbd08be1c973a60c3c077cf3e29e7c.JPEG)\n\n我们为什么最终选择了 ShardingSphere 呢？主要从这几个因素考虑：\n\n#### **技术环境**\n\n-   我们团队是 Java 体系下的，对 Java 中间件有一些偏爱\n    \n-   更偏向于轻量级组件，可以深入研究的组件\n    \n-   可能会需要一些个性定制化\n    \n\n#### **专业程度**\n\n-   取决于中间件由哪个团队进行维护，是否是名师打造，是否是行业标杆\n    \n-   更新迭代频率，最好是更新相对频繁，维护较积极的\n    \n-   流行度问题，偏向于流行度广、社区活跃的中间件\n    \n-   性能问题，性能能满足我们的要求\n    \n\n#### **使用成本**\n\n-   学习成本、入门成本和定制改造成本\n    \n-   弱浸入性，对业务能较少浸入\n    \n-   现有技术栈下的迁移成本，我们当前技术栈是 SSM 体系下\n    \n\n#### **运维成本**\n\n-   高可用、高稳定性\n    \n-   减少硬件资源，不希望再单独引入一个代理中间件，还要考虑运维成本\n    \n-   丰富的埋点、完善的监控\n    \n\n## **四、业务实践**\n\n在业务实践中，我们经历了从新库新表的设计，分库分表自建代理、服务收口、上游订单应用迁移，历史数据迁移等过程。\n\n### **4.1 新表模型**\n\n为了建立分库分表下的关联关系，和更加合理有效的结构，我们新申请了订单分库分表的几个库，设计了一套全新的表结构。表名以年份结尾、规范化表字段、适当增删了部分字段、不使用自增主键关联，采用业务唯一键进行关联等。\n\n表结构示例如下图：\n\n![](https://oscimg.oschina.net/oscnet/up-12ac37720f2e42ee55255c35d626231b6e2.png)\n\n### **4.2 服务收口**\n\n自建了一个分库分表数据库的服务代理 Dal-Sharding。每一个需要操作订单库的服务，都要通过代理服务进行操作数据库，达到服务的一个收口效果。同时，屏蔽了分库分表的复杂性，规范数据库的基本增删改查方法。\n\n![](https://oscimg.oschina.net/oscnet/up-c52f69ac309c952856c373a629d4143440f.png)\n\n### **4.3 平滑过渡**\n\n应用迁移过程中，为了保证应用的平滑过渡，我们新增了一些同步逻辑，来保证应用的顺利迁移，在应用迁移前后，对应用没有任何影响。未迁移的应用，可以读取到迁移后应用写入的订单数据；迁移后的应用，能读取到未迁移应用写入的订单数据。同时，统一实现了此逻辑，减少各个应用的迁移成本。\n\n**新老库双读**\n\n顾名思义，就是在读取的时候，两个库可能都要进行读取，即优先读取新库，如果能读到记录，直接返回；否则，再次读取老库记录，并返回结果。\n\n双读的基本过程如下：\n\n![](https://oscimg.oschina.net/oscnet/up-f1690cb1265e12a4523ea7aaf8f55c21620.png)\n\n新老库双读，保证了应用迁移过程中读取的低成本，上游应用不需要关心数据来源于新的库还是老的库，只要关心数据的读取即可，减少了切换新库和分库分表的逻辑，极大的减少了迁移的工作量。\n\n实践过程中，我们通过切面实现双读逻辑，将双读逻辑放入到切面中进行，减小新库的读取逻辑的侵入，方便后面实现对双读逻辑的移除调整。\n\n同时，新增一些配置，比如可以控制到哪些表需要进行双读，哪些表不需要双读等。\n\n**新老库双写**\n\n新老库双写，就是在写入新库成功后，异步写入到老库中。双写使得新老库都同时存在这些订单数据，尚未迁移通过代理服务操作数据库的应用得以正常的运作。\n\n双写的基本过程如下：\n\n![](https://oscimg.oschina.net/oscnet/up-2e4a7924da6ac7e366b115ad3b0eaceb9b5.png)\n\n双写其实有较多的方案，比如基于数据库的日志，通过监听解析数据库日志实现同步；也可以通过切面，实现双写；还可以通过定时任务进行同步；另外，结合到我们自己的订单业务，我们还可以通过订单事件（比如创单成功、出票成功、退票成功等），进行双写，同步数据到老库中。\n\n目前，我们经过考虑，没有通过数据库日志来实现，因为这样相当于把逻辑下沉到了数据库层面，从实现上不够灵活，同时，可能还会涉及到一些权限、排期等问题。实践中，我们采取其他三种方式，互补形式，进行双写。异步切面双写，保证了最大的时效性；订单事件，保证了核心节点的一致性；定时任务，保证了最终的一致性。\n\n跟双读一样，我们也支持配置控制到哪些表需要进行双写，那些表不需要双写等。\n\n**过渡迁移**\n\n有了前面的双读双写作为基础，迁移相对容易实行，我们采取逐个迁移的方式，比如，按照服务、按照渠道和按照供应进行迁移，将迁移工作进行拆解，减少影响面，追求稳健。一般分为三步走方式：\n\n1）第一阶段，先在新对接的供应商中进行迁移新库，因为新上线的供应商，订单量最少，同时哪怕出现了问题，不至于影响到之前的业务。\n\n2）再次迁移量比较少的线上业务，此类订单，有一些量，但是追求稳定，不能因为切换新库而产生影响。所以，将此类业务放到了第二阶段中进行。\n\n3）最后一步是，将量较大的业务，逐渐迁移到新库中，此类业务，需要在在有前面的保证后，方能进行迁移，保证订单的正常进行。\n\n![](https://oscimg.oschina.net/oscnet/up-60ddd27b3aaf8d0451b641ca02afcfad65b.png)\n\n### **4.4 数据迁移**\n\n数据迁移，即将数据，从老库迁移到新库，是新老库切换的一个必经过程。迁移的常规思路，一般是每个表一个个进行迁移，结合业务，我们没有采取此做法，而是从订单维度进行迁移。\n\n举个例子：假如订单库有 Order 表、OrderStation 表、OrderFare 表三个表，我们没有采取一个一个表分别进行迁移，而是根据订单号，以每一个订单的信息，进行同步。\n\n大致过程如下：\n\n1）开启一个定时任务，查询订单列表，取得订单号等基本订单信息。\n\n2）根据这个订单号，去分别查询订单的其他信息，取得一个完整的订单信息。\n\n3）校验订单是否已经完成同步，之前完成同步了则直接跳过，否则继续执行下一个订单号。\n\n4）将老库的完整的订单信息，映射成新库的对应的模型。\n\n5）将新的订单信息，同步写入到新库各个表中。\n\n6）继续执行下一个订单号，直到所有的订单号都完全同步结束。\n\n![](https://oscimg.oschina.net/oscnet/up-c900164b1e0238e7f40b0acfd8cac4fbf8b.png)\n\n### **4.5 完成效果**\n\n订单库经过一个全新的重构，目前已经在线上稳定运行，效果显著，达到了我们想要的效果。\n\n-   服务收口，将分库分表逻辑，收口到了一个服务中；\n    \n-   接口统一管理，统一对敏感字段进行加密；\n    \n-   功能灵活，提供丰富的功能，支持定制化；\n    \n-   分库分表路由透明，且基于主流技术，易于上手；\n    \n-   完善的监控，支持到表维度的监控；\n    \n\n## **五、****常见问题总结**\n\n### **5.1 分库分表典型问题**\n\n**问题1**：如何进行跨库操作，关联查询，跨库事务？\n\n**回答：**对于跨库操作，在订单主流程应用中，我们目前是禁止了比如跨库查询、跨库事务等操作的。对于跨库事务，因为根据订单号、创建年份路由，都是会路由到同一个数据库中，也不会存在跨库事务。同样对于跨库关联查询，也不会存在，往往都是根据订单来进行查询。同时，也可以适当进行冗余，比如存储车站编码的同时，多存储一个车站名称字段。\n\n**问题2**：如何进行分页查询？\n\n**回答**：目前在订单主流程应用中的分页查询，我们直接采用了 Sharding-JDBC 提供的最原始的分页方式，直接按照正常的分页 SQL，来进行查询分页即可。理由：主流程订单服务，比如出票系统，往往都是查询前面几页的订单，直接查询即可，不会存在很深的翻页。当然，对于要求较高的分页查询，可以去实现二次查询，来实现更加高效的分页查询。\n\n**问题3**：如何支持很复杂的统计查询？\n\n**回答**：专门增加了一个宽表，来满足那些很复杂查询的需求，将常用的查询信息，全部落到此表中，进而可以快速得到这些复杂查询的结果。\n\n### **5.2 API 方法问题**\n\n**问题：**服务收口后，如何满足业务各种不同的查询条件？\n\n**回答：**我们的 API 方法，相对固定，一般查询类只有两个方法，根据订单号查询，和根据 Condition 查询条件进行查询。对于各种不同的查询条件，则通过新增 Condition 的字段属性来实现，而不会新增各种查询方法。\n\n### **5.3 均匀问题**\n\n**问题：**在不同 group 中，数据会存在分布不均匀，存在热点问题？\n\n**回答：**是的，比如运行 5 年后，我们拓展成了 3 个 group，每一个 group 中存在 3 个库，那么此时，读写最多的应该是第三个 group。不过这种分布不均匀问题和热点问题，是可接受的，相当于前面的两个 group，可以作为历史归档 group，目前主要使用的 group 为第三个 group。\n\n随着业务的发展，你可以进行调配，比如业务发展迅速，那么相对合理的分配，往往不会是每个 group 是 3 个库，更可能是应该是，越往后 group 内的库越多。同时，因为每个 group 内是存在多个库，与之前的某一个库的热点问题是存在本质差别，而不用担心单数据库瓶颈问题，可以通过加库来实现扩展。\n\n### **5.4 Group 内路由问题**\n\n**问题：**对于仅根据订单号查询，在 group 内的路由过程是读取 group 内所有的表吗？\n\n**回答：**根据目前的设计，是的。目前是按年份分组，订单号不会存储其他信息，采用携程统一方式生成，也就是如果根据订单号查询，我们并不知道是存在于哪个表，则需要查询 group 内所有的表。对于此类问题，通常推荐做法是，可以适当增加因子，在订单号中，存储创建年份信息，这样就可以知道对应那个表了；也可以年份适当进行延伸，比如每 5 年一次分表，那么这样调整后，一个 group 内的表应该相对很少，可以极大加快查询效能。\n\n### **5.5 异步双写问题**\n\n**问题**：为什么双写过程，采用了多种方式结合的方式？\n\n**回答**：首先，切面方式，能最大限度满足订单同步的时效性。但是，在实践过程中，我们发现，异步切面双写，会存在多线程并发问题。因为在老库中，表的关联关系依赖于数据库的自增 ID，依赖于表的插入顺序，会存在关联失败的情况。所以，单纯依靠切面同步还不够，还需要更加稳健的方式，即定时任务（订单事件是不可靠消息事件，即可能会存在丢失情况）的方式，来保证数据库的一致性。\n\n**关于作者**\n\n瑞华：携程火车票研发团队，负责火车票业务的开发以及创新。火车票研发在多种交通线路联程联运算法、多种交通工具一站式预定、高并发方向不断地深入探索和创新，持续优化用户体验，提高效率，致力于为全球人民买全球火车票。\n\n","date":"2022-11-01","author":"瑞华","excerpt":"订单库经过一个全新的重构，目前已经在线上稳定运行，效果显著，达到了我们想要的效果。","metaDescription":null,"createdAt":"2022-11-03T09:11:41.372Z","updatedAt":"2022-11-03T09:11:42.622Z","publishedAt":"2022-11-03T09:11:42.619Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":33,"attributes":{"feature":true,"title":"覆盖 70% 核心业务，ShardingSphere 如何成为喜马拉雅架构演进的催化剂","content":"### 背景\n\n喜马拉雅成立之初，各个业务管理各自的数据库、缓存，各个业务都要了解中间件的各种部署情况，导致业务间的合作，需要运维、开发等方面的人工介入，效率较低，扩展困难，安全风险也很高，资源利用率也不高。喜马拉雅在发展中，逐渐意识到需要在公司层面，提供统一的定制化的数据访问平台的重要性。为此，我们推出了自己的 PaaS 化平台，PaaS 化就是对资源的使用做了统一的入口，业务只需要申请一个资源 ID，就能使用数据库，达到对资源使用的全部系统化，其中对数据库的访问我们基于 Apache ShardingSphere 来实现，并基于 Apache ShardingSphere 强大功能做些优化和增强。\n\n### 整体架构\n\n我们 PaaS 平台建设中，负责和数据层通信的 dal 层中间件我们叫 Arena，其中对数据库的访问我们叫 Arena-Jdbc。\n\nArena-Jdbc 层的能力基本是基于 Apache ShardingSphere 的能力建设，我们只是基于喜马拉雅需要的特性做了增强和优化，整体架构如下：\n\n![](https://img-blog.csdnimg.cn/img_convert/96d0bfa8558f18cfd3ce1544cae6f58d.png)​\n\n### Pull Frame\n\nConsul Pull Frame 是我们对 Consul 的配置自动拉起封装为统一的 Pull 框架，我们除了数据库，还有缓存，每种还有不同的使用方式，我们对不同的使用方式只需要实现对应的实现类和初始化，更新，切好这些接口就行，框架会统一把解析好的数据给到，具体一种场景不需要关心和 Consul 的交互，为后面的资源 PaaS 化提供了简单的接入能力。\n\n### 故障容灾\n\n· 自动重连\n\n我们对故障容灾在设计时就考虑了平时通用的一些故障场景，比如数据库 server 挂了，我们做自动重链，不需要业务做重启操作。\n\n· 本地快照\n\n本地快照是为了防止 Consul 不可用时，业务不能启动，所以我们在拉到远程配置后，会本地存储一份，在拉配置时，如果远程失败，就用本地的配置，保证 Consul 挂了，不影响业务，每次拉到新的配置时，会更新本地的快照。\n\n· 灰度更新\n\n灰度更新是为了支持配置变更时找灰度的逻辑，对于数据库层面的变更，是非常危险的，如果一下就全量变更，有可能会触发线上事故，所以通过灰度变更的机制，业务可以先选择一个容器实例来变更，没有问题后，再全量变更，把风险降到最低。\n\n· 密码安全\n\n没有 PaaS 化之前，我们的数据库密码都是 DBA 统一管理的，但 PaaS 化后，访问数据库的密码就存在配置文件中，如果明文，就太不安全，所以我们对密码统一做了加密处理，在 Arena-Jdbc 层统一做解密，确保密码不会泄露出去。\n\n### 统一数据源\n\n为了让业务做最低成本的改造，Arena-Jdbc 需要提供一个统一的数据源，不论上层用什么框架，不影响业务只需要替换数据源接入即可，对于数据库连接池我们默认使用 HikariCP DataSource 也支持个性化的业务，业务可以通过配置指定连接池。\n\n我们基于 Apache ShardingSphere 的连接池封装了一个我们自己的 DataSource，我们叫 ArenaDataSource，通过 ArenaDataSource 封装了各种不同场景聚合的使用一个 ArenaDataSource 支持三种使用方式：\n\n1.  支持原生直接连接\n2.  支持 Proxy 模式，也是 Apache ShardingSphere 的 Proxy\n3.  支持直接连接分库分表\n\n业务只需要一个 DataSource，即支持分库分表，也支持简单的直接连接的模式，这样的好处是业务以后要分库分表，就不再需要升级中间件了，为了彻底解决业务升级的成本，我们做了配置自动升级，就是你之前是简单直接链接使用，为了 PaaS 化，后来业务发展了，需要分库分表了，以及从分库分表需要多活部署了，这些都不要再升级依赖了，只需要配置动即可。\n\n### 资源动态变更\n\n资源动态变更是 PaaS 平台基本的能力，接入 PaaS 后，业务修改数据库的任何属性，都不再需要业务方代码变更，重新发布。\n\nApache ShardingSphere 也支持数据库属性的动态变更，我们基于自己的内部系统的特征，实现了基于 Consul 的资源变更通知，我们的资源存在 Consul。\n\nArena-Jdbc 支持对使用的资源做无损的变更，Arena-Jdbc 收到资源变更时，会先对新下发的资源做预热处理，预热后，再切换使用的数据源，切换成功后，再销毁老的数据源，业务无感知。\n\n如果新的资源预热失败，则不会做变更处理，保证下发的资源是可用的，规避错误下发的问题。\n\n扩容和缩容也是同理，一期数据需要运维手动迁移，迁移好了后，直接在 PaaS 平台下发新的配置即可，二期支持自动迁移数据和配置变更结合。\n\n同时支持 Proxy 的无损上下线机制，通过 PaaS 平台对 Proxy 的变更，把需要下线的 Proxy 节点去掉，通知 Arena-Jdbc，Arena-Jdbc 会把缩容的 Proxy 节点去掉，做到无损下线。\n\n### 读写分离\n\n读写分离我们完全基于 Apache ShardingSphere 的来实现，我们根据喜马拉雅业务的特性，对强制路由做了增强，不需要规则配置为 Hint 模式，只要线程上下文带有强制路由的标志，就可以路由到指定的库和表，不受分表规则的影响，我们重写了 ShardingStandardRoutingEngine 的 Sharding 时路由库和表的逻辑：\n\nprivate Collection<String> routeDataSources(final TableRule tableRule, final ShardingStrategy databaseShardingStrategy, final List<ShardingConditionValue> databaseShardingValues) {\n        //先判断是否存在Hint上下文路由标，如果有,则优先根据用户指定的规则路由库\n        Collection<Comparable<?>> databaseShardings = HintManager.getDatabaseShardingValues(tableRule.getLogicTable());\n        if (databaseShardings != null && databaseShardings.size() > 0) {\n            List<String> list = new ArrayList<>(4);\n            for (Comparable<?> databaseSharding : databaseShardings) {\n                list.add((String) databaseSharding);\n            }\n            if (log.isDebugEnabled()) {\n                log.debug(\"route dataSources, find HintManager, so hint to: {}\", list);\n            }\n            return list;\n        }\n        //没有Hint路由规则，则按Sharding 规则路由\n        if (databaseShardingValues.isEmpty()) {\n            return tableRule.getActualDatasourceNames();\n        }\n        Collection<String> result = new LinkedHashSet<>(databaseShardingStrategy.doSharding(tableRule.getActualDatasourceNames(), databaseShardingValues, properties));\n        Preconditions.checkState(!result.isEmpty(), \"no database route info\");\n        Preconditions.checkState(tableRule.getActualDatasourceNames().containsAll(result), \n                \"Some routed data sources do not belong to configured data sources. routed data sources: `%s`, configured data sources: `%s`\", result, tableRule.getActualDatasourceNames());\n        return result;\n    }\n\n路由表也是同样的逻辑，我们重写了 ShardingStandardRoutingEngine 的 routeTables 方法，和上面一样，先从 Hint 的上下文获取。这样通过上下文的方式能很好地满足业务个性化的路由规则，能和 Sharding 规则共存。\n\n### Database Plus\n\nApache ShardingSphere 除了提供基本的分库分表，读写分离的能力外，在上层还提供了很多的插件和扩展的机制，这让我们在基于数据库提供更偏向业务的能力非常容易，成本非常低，这叫 Database Plus。\n\nDatabase Plus 简单的说就是用 Apache ShardingSphere 的数据库中间件，不仅仅是提供了分库分表这一基本能力，通过对底层数据的封装为统一的交互标准插件模式，可以在上面实现很多业务的通用的场景的需求，比如喜马拉雅除了用到 Apache ShardingSphere 基础的能力外，我们也享受了 Database Plus 的威力，我们在它的基础上轻松实现了支持压测的影子库和影子表，数据加解密，机房级别容灾的同城双读，分布式唯一 ID。\n\n### 影子库和影子表\n\n影子库影子表我们对 Apache ShardingSphere 做了改动，Apache ShardingSphere 需要修改 SQL，我们认为对业务有改造成本，同时结合我们自己的压测平台，我们和业界一样，我们也实现了影子标记，通过全链路压测标的传递来判断是否路由到影子库/影子表，业务无需任何改造，即可使用影子库影子表来做压测，同时不需要在运行时对 SQL 改写，提升了性能，我们重写了 ShadowSQLRouter。\n\npublic class ArenaShadowSQLRouter extends ShadowSQLRouter {\n    @Override\n    public boolean isShadow(final SQLStatementContext<?> sqlStatementContext, final List<Object> parameters, final ShadowRule rule) {\n        if (sqlStatementContext instanceof InsertStatementContext || sqlStatementContext instanceof WhereAvailable\n            || sqlStatementContext instanceof UpdateStatementContext) {\n                 //这里就是判断是否有压测标，如果有，ShardingSphere则会找影子的逻辑。\n                return ArenaUtilities.checkPeakRequest();\n        }\n        return false;\n    }\n}\n\n通过 spi 的方式把我们自定义的 ArenaShadowSQLRouter 给 Apache ShardingSphere 加载使用，不得不说 Apache ShardingSphere 的插件设计很赞，很方便自定义和扩展。\n\n配置还是和 Apache ShardingSphere 的一样：\n\n配置影子库规则\n\\- !SHADOW\n    # true-影子表，false-影子库(默认)\n    enableShadowTable: true\n\n    # 源库名称(对应DataSources数据源配置中的名称)，影子库才需要配，影子表不需要配置\n    sourceDataSourceNames:  \n      \\- ds0           # 源库，与影子库shadow_ds0对应\n      \\- ds1           # 源库，与影子库shadow_ds1对应\n\n    # 影子库名称(对应dataSources数据源配置中的名称)，影子库才需要配，影子表不需要配置\n    shadowDataSourceNames:  \n      \\- shadow_ds0    # 影子库，与源库ds0对应\n      \\- shadow_ds1    # 影子库，与源库ds1对应\n\nenableShadowTable 我们新增了该属性，来确定是使用影子库还是影子表。\n\n### 影子库\n\n影子库，一定要填 sourceDataSourceNames 和 shadowDataSourceNames，enableShadowTable 不用设置，或者设置为 false。\n\nsourceDataSourceNames\n\n按顺序映射，一一对应：\n\nds --> shadow_ds\n\nds1--> shadow_ds1\n\n影子库/影子表是最后一个路由规则，如果发现有影子库/影子表，则根据实际的库找到对应的影子库/影子表，执行 SQL。\n\n### 同城多活\n\n基于喜马拉雅的业务特性，读多写少，我们只实现了对读业务的双机房部署，写业务还是路由到主机房。\n\n为了增强容灾能力，喜马拉雅搭建了双机房，同时承载业务流量，当一个机房故障时，可以把流量快速切换到另一个机房，我们在 dal 层设计上支持双写，这里充分利用了 Apache ShardingSphere 的读写分离功能，读和写可以配置独立的数据源，我们只需要在上面做了一层封装，在切换时候动态变更对应的数据源即可，为了切换时不影响业务的流量，我们是先预热新的数据源，再销毁老的数据源。\n\n架构图如下：\n\n![](https://img-blog.csdnimg.cn/img_convert/19f6cb8d352c0b53c2cd7b5ba0ec9003.png)​\n\n另外我们也对双写做了研究和探索，关键在于数据库的双向同步，基于阿里开源的 otter 做了改造，支持基于 gtid 模式同步，不依赖打标，打标会有性能开销，在一些业务做了试用。\n\n### 分布式唯一 ID\n\n分库分表后，唯一 id 是必须要满足的需求，Apache ShardingSphere 默认提供了 snowfake 和 uuid 算法，但不是很 db 时候的场景，db 需要保证顺序和格式，所以我们基于 Apache ShardingSphere 提供的接口，也实现自己的唯一 id 生成策略：数据分片后，不同 MySQL 实例生成全局唯一主键是非常棘手的问题。Arena-Jdbc 实现了 Apache ShardingSphere 的分布式主键生成器接口，通过集成喜马拉雅内部的全局唯一 id 生成服务，提供了适用于喜马拉雅内部的自增主键生成算法- BoushId 主键生成策略。\n\n### 监控和报警\n\n做一个数据库中间件，监控是必不可少的部分，就像我们的眼睛，没有监控就是瞎抓，以及对异常情况的报警也是非常重要的部分，只有完善的监控和报警才能算是一个完整的产品，得益于 Apache ShardingSphere 在设计时就提供了钩子，我们能非常小的成本就能实现对 SQL 层面的监控和报警。\n\nArena-Jdbc 客户端通过钩子回调，从多维度数据来分析使用数据库的运行情况，以 30s 为一次统计周期，每个周期统计的数据包括：MySQL 总请求量，新增、删除、修改和查询的请求量，失败的请求量和慢请求量，影子库的流量，以及统计响应时间的 TP 百分比，还有连接池的等待时间、建连时间、连接数等信息。这些指标会发送给专门的收集指标服务，并持久化到时序数据库，PaaS 平台可以从时序数据库中查询数据，展示给各个业务，对于异常 SQL 和慢 SQL，做报警等后续处理。\n\n![](https://img-blog.csdnimg.cn/img_convert/104ee654eee8dc89263510fa9f31aad6.png)​\n\n### 其他\n\n我们除了基于 Apache ShardingSphere 实现上述关键特性外，我们还对 Apache ShardingSphere 做了一些优化和改进，以更适合喜马拉雅的业务。\n\n· 优化分片规则，启动时，如果分片的真实表不存在的情况则报错，将配置错误前置；\n\n· 有的业务方有几百，甚至几千的分表，这种情况下，由于 Apache ShardingSphere 中的联邦查询需要依次扫表，启动速度很慢，达到了分钟级别。针对这种情况，我们新增了 props 配置项，不再初始化联邦查询，大大加快了启动速度，并且在使用中，也没有用联邦查询；\n\n· 优化了 Apache ShardingSphere，执行 SQL 异常不报错误的情况；\n\n· 由于有的业务方，对重要的表采用了大写的表名和列名，我们去掉 Apache ShardingSphere 中，对配置中的大写的表名列名强制小写的情况，允许大写的表名和列名；\n\n· 新增了 props 配置项，可以调节 Apache ShardingSphere 的编译缓存的大小；\n\n· 优化 Apache ShardingSphere 复合分片算法，精确匹配分片字段；\n\n· 在 ComplexShardingStrategyConfiguration 中，添加 shardingColumnList 字段，修复 Apache ShardingSphere 批量 insert 不返回主键的问题，这个问题在 mybatis-plus 中比较常见；\n\n· 不分片的表，支持使用默认的主键 id 生成策略。\n\n### 总结\n\n基于 Apache ShardingSphere 实现的数据库中间件 Arena-Jdbc，经过半年的时间，已经覆盖了喜马拉雅的 70% 的核心业务，目前没有发现任何问题，表现的非常稳定，通过和我们的 PaaS 平台结合，业务也非常愿意接入，另外我们使用 Apache ShardingSphere 时，社区还没有发布 stable 的版本，所以我们在使用过程中也遇到了些问题，基本上我们都解决了，有的社区也有对应的解决方案，得益于社区非常活跃，我们以后也希望把我们做的一些 feature 能回馈到社区，为 Apache ShardingSphere 的发展做出一点点小贡献。\n\n非常感谢基础架构团队胡建华、彭荣新提出的宝贵意见和建议，感谢喜马拉雅基础小伙伴们在项目推广过程中的大力支持，让 Apache ShardingSphere 在喜马拉雅生根发芽。\n\n非常感谢亮哥亲自来喜马拉雅对 Apache ShardingSphere 的技术内幕和规则做了一次全面的分享，非常关心我们在使用 Apache ShardingSphere 过程中遇到的问题，在现场对小伙伴提的问题都一一作了深入的解答，非常感谢亮哥，祝 Apache ShardingSphere 越来越好。","date":"2022-11-01","author":"沈辉 ","excerpt":"基于 Apache ShardingSphere 强大功能做优化和增强","metaDescription":null,"createdAt":"2022-11-03T09:45:09.484Z","updatedAt":"2022-11-03T09:45:10.507Z","publishedAt":"2022-11-03T09:45:10.504Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":37,"attributes":{"feature":true,"title":"ShardingSphere 在数十个联通政务热线场景中的应用：稳定、高效、可复制","content":"# **一、项目背景**\n\n  随着“互联网+政务服务”政策的持续发力，越来越多的地方政府开始着手打造“互联网+”环境下的政务服务平台。\n\n  近年来，联通数科先后助力数十个政务热线平台上线，以提升政务服务能力，部分项目如下图。\n![up-20ec9e57d65c64dcf55a4277105b5ff0488.webp](https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/up_20ec9e57d65c64dcf55a4277105b5ff0488_6143b4132c.webp)\n\n  政务热线业务有并发高、数据量大的特点，我们每拨打一次政务热线，即产生一条工单记录。相比以往，疫情期间的热线业务量更是翻了数倍。\n\n  **在政务热线的工单模块中，为满足大规模、海量数据以及高稳定的业务需求，联通数科采用 ShardingSphere 对工单信息进行分片存储，在政民互动的桥梁中构建坚固的桥墩。**\n\n# **二、业务挑战**\n\n  政务热线是政府与企业、群众互动的主要渠道，以单一号码面对公众，提供 7*24 服务。除受理工单外，还包括了话务、知识库、语音对答及智能坐席等业务。\n\n  在系统规划阶段，数据库层作为业务系统的基石，技术选型过程更为谨慎。系统的稳定是硬性指标，其次是性能，性能直接影响热线业务的服务能力，同时还要具备易于维护和管理的特点，便于系统升级及后台维护。\n\n  在数据库架构选型中，政务热线系统对架构要求如下：\n\n  - **成熟稳定**\n  - **高性能**\n  - **易维护**\n  - **业务代码低耦合**\n\n# **三、选择 ShardingSphere 的理由**\n\n  **经过技术团队多轮调研测试，包括稳定性、产品功能、接入方式及性能等，最终选择“ShardingSphere + MySQL”分布式数据库解决方案。**\n\n  - **理念先进**\n\n  随业务应用场景多元化发展，数据库已呈碎片化趋势，**「Database Plus」**旨在数据库上层建立标准和生态，为数据库提供增量能力，可防止数据库绑定、实现自主可控，突破原有数据库功能局限，提升数据基础设施整体功能上限。\n\n  - **成熟稳定/案例多**\n\n  ShardingSphere 于 2016 年开源，经过多年的研发迭代，产品历经多个头部互联网大型场景打磨，成熟稳定、值得信赖。\n\n  - **功能全面**\n\n  ShardingSphere 除了具备 Sharding 能力外，数据加解密和影子库的功能也是作为联通技术团队评估的指标，通过一套技术栈既可解决安全和压测的场景需求。ShardingSphere 为政务热线系统建设及架构扩展，提供了更多的能力支持。\n\n  - **高兼容/接入方便**\n\n  ShardingSphere 与 MySQL 协议兼容，语法支持度也非常高，在开发过程中几乎不用考虑 SQL 的支持问题，接入非常方便。\n\n  - **高性能**\n\n  政务热线忙时有数千坐席在线，工单记录以千万记录为起始单位，对性能有明确的要求。ShardingSphere-JDBC 定位为轻量级的 Java 框架，压测结果数据理想，可满足政务系统服务能力要求。\n\n  - **易运维**\n\n  在 JDBC 良好的性能基础之上，ShardingSphere 还提供了对运维人员友好的 Proxy，使用常用的客户端即可直接访问，上手维护较容易。\n\n  除了以上五个维度的权衡，选型中也参考了其他数据库中间件产品，以下表格是 ShardingSphere 和 Mycat 的比对。\n\n  |              | **Apache ShardingSphere**                   | **Mycat**                                     |\n  | ------------ | ------------------------------------------- | --------------------------------------------- |\n  | 开源协议     | Apache-2.0                                  | GPL-2.0/GPL-3.0                               |\n  | 接入端       | 客户端/代理端                               | 代理端                                        |\n  | 数据库支持   | 客户端：任意代理端：MySQL/PG/openGauss      | MySQL、Oracle、DB2、SQL Server、PostgreSQL 等 |\n  | 性能         | 客户端：损耗低代理端：损耗高                | 损耗高                                        |\n  | 读写分离     | 有                                          | 有                                            |\n  | 数据加密     | 有                                          | 无                                            |\n  | 全链路压测   | 有                                          | 无                                            |\n  | 社区成熟度   | Apache 软件基金会信通院可信认证社区活跃度高 | 个人/三方社区社区活跃度一般                   |\n  | 商业公司支持 | 有                                          | 无                                            |\n\n# **四、解决方案**\n\n  由于“ShardingSphere+MySQL ”具备稳定、易用及性能极致等特点，该方案在联通数科的多个政务热线项目中复制使用。\n\n  - **部署方式**\n\n  ShardingSphere-JDBC 和 ShardingSphere-Proxy 混合使用的方式，兼顾了系统性能和可维护性。\n\n  - **分片键/分片算法**\n\n  政务热线的工单表相对较大，且使用逻辑简单，因此在分片设计上，选取工单 ID 作为分片键，通过哈希算法分布数据。\n\n  - **迁移**\n\n  新建项目不涉及历史数据，因此无需考虑数据迁移环节，上线即直接使用。\n\n  - **实施过程**\n\n  回忆数个已上线的项目，由于技术方案评估较全面，且多次复制方案，实施过程并没有遇到“坑”。\n\n  - **系统架构**\n\n  政务热线业务模块采用了微服务架构，节点在十几到几十不等，以具体省市规模决定。在数据库层，使用物理机部署 3 节点 MHA，每个政务热线系统一般需要 4 套 MHA 来支撑业务。\n\n![up-6eb6faa996c81e6de688a2dd9e204f730e6.webp](https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/up_6eb6faa996c81e6de688a2dd9e204f730e6_e867e826e4.webp)\n\n# **五、用户收益**\n\n  - **业务角度**\n    - 支持客户端模式，最大化保证了政务热线对性能的要求；\n    - 系统稳定，为业务连续性带来有力的支撑。\n  - **研发角度**\n    - 兼容良好，兼容主流数据库 SQL 语法，对接即可使用；\n    - 功能全面，读写分离、数据加密及全链路压测方案，系统扩展性强。\n  - **运维角度**\n    - 支持代理端模式，为运维人员提供了良好的维护入口。\n  - **技术选型角度**\n    - 防止数据库绑定，为未来升级提供了足够的灵活度支撑。\n\n# **六、总结**\n\n  通过联通数科政务热线的多个案例，验证了 ShardingSphere 在政务场景的能力，也进一步说明了 ShardingSphere 无行业限制的特点。\n  目前我国数字政府建设已进入全面提速阶段，联通数科和 ShardingSphere 通力合作，希望助力更多政务热线项目落地，打造更多高效、便捷和安全的政务热线。","date":"2022-07-11","author":"SphereEx","excerpt":"联通数科整合中国联通“云、大、物、智、链、安”（云计算、大数据、物联网、人工智能、区块链、安全）等能力，面向智慧城市、数字政府、工业互联网、生态环保、文化旅游等领域，联通数科已积累大量行业标杆案例，成功为客户提供更加丰富、专业的产品和服务。","metaDescription":null,"createdAt":"2023-03-28T07:23:33.986Z","updatedAt":"2023-08-25T06:21:38.774Z","publishedAt":"2023-03-28T07:23:44.797Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":41,"attributes":{"feature":true,"title":"喜马拉雅基于Apache ShardingSphere实践","content":"# **背景** \n\n \n\n喜马拉雅成立之初，各个业务管理各自的数据库、缓存，个业务都要了解中间件的各种部署情况，导致业务间的合作，需要运维、开发等方面的人工介入，效率较低，扩展困难，安全风险也很高，资源利用率也不高。喜马拉雅在发展中，逐渐意识到需要在公司层面，提供统一的定制化的数据访问平台的重要性。为此，我们推出了自己的PaaS化平台，PaaS化就是对资源的使用做了统一的入口，业务只需要申请一个资源ID，就能使用数据库，达到对资源使用的全部系统化，其中对数据库的访问我们基于 Apache ShardingSphere 来实现，并基于 Apache ShardingSphere 强大功能做些优化和增强。\n\n \n\n# **整体架构**\n\n \n\n我们PaaS平台建设中，负责和数据层通信的dal层中间件我们叫Arena,其中对数据库的访问我们叫Arean-Jdbc\n\nArena-Jdbc 层的能力基本是基于 Apache ShardingSphere 的能力建设，我们只是基于喜马拉雅需要的特性做了增强和优化,整体架构如下：\n\n \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=NmU5ZDNhMjRlNDczZjdiYzg3NTE0Yzg4OTc5N2Q0M2ZfUzg3c3FCbkozTG0zUzZDR1dFb1dHT29Od3Nod0VWQmRfVG9rZW46Ym94Y25UY1VJQlcxeG1qZ2RVU01aUlIwQjdmXzE2ODAwNzcwNDk6MTY4MDA4MDY0OV9WNA)\n\n \n\n# **Pull Frame**\n\n \n\nConsul Pull Frame 是我们对Consul 的配置自动拉起封装为统一的Pull框架，我们除了数据库，还有缓存，每种还有不同的使用方式，我们对不同的使用方式只需要实现对应的实现类和初始化，更新，切好这些接口就行，框架会统一把解析好的数据给到，具体一种场景不需要关心和Consul的交互，为后面的资源PaaS化提供了简单的接入能力。\n\n \n\n# **故障容灾**\n\n \n\n***· 自动重连***\n\n我们对故障容灾在设计时就考虑了平时通用的一些故障场景，比如数据库server 挂了，我们最做自动重链，不需要业务做重启操作。\n\n \n\n***· 本地快照***\n\n本地快照是为了防止Consul不可用时，业务不能启动，所以我们在拉到远程配置后，会本地存储一份，在拉配置时，如果远程失败，就用本地的配置，保证Consul挂了，不影响业务，每次拉到新的配置时，会更新本地的快照。\n\n \n\n***· 灰度更新***\n\n灰度更新是为了支持配置变更时找灰度的逻辑，对于数据库层面的变更，是非常危险的，如果一下就全量变更，有可能会触发线上事故，所以通过灰度变更的机制，业务可以先选择一个容器实例来变更，没有问题后，再全量变更，把风险降到最低。\n\n \n\n***· 密码安全***\n\n没有PaaS化之前，我们的数据库密码都是dba统一管理的，但PaaS化后，访问数据库的密码就存在配置文件中，如果明文，就太不安全，所以我们对密码统一做了加密处理，在 Arean-Jdbc 层统一做解密，确保密码不回泄露出去。\n\n \n\n# **统一数据源**\n\n \n\n为了让业务做最低成本的改造，Arena-jdbc 需要提供一个统一的数据源，不论上层用什么框架，不影响,业务只需要替换数据源接入即可，对于数据库连接池我们默认使用 HikariCP  DataSource 也支持个性化的业务，业务可以通过配置指定连接池。\n\n我们基于Apache ShardingSphere 的连接池封装了一个我们自己的DataSource,我们叫ArenaDataSource，通过ArenaDataSource封装了各种不同场景聚合，的使用一个 ArenaDataSource 支持三种使用方式：\n\n1. 支持原生直接连接\n\n2. 支持Proxy模式，也是Apache ShardingSphere的proxy\n\n3. 支持直接连接分库分表\n\n业务只需要一个 DataSource，即支持分库分表，也支持简单的直接连接的模式，这样的好处是业务以后要分库分表，就不再需要升级中间件了，为了彻底解决业务升级的成本，我们做了配置自动升级，就是你之前是简单直接链接使用，为了PaaS化，后来业务发展了，需要分库分表了，以及从分库分表需要多活部署了，这些都不要再升级依赖了，只需要配置动即可。\n\n\n\n# **资源动态变更**\n\n \n\n资源动态变更是PaaS平台基本的能力，接入PaaS后，业务修改数据库的任何属性，都不再需要业务方代码变更，重新发布 \n\nApache ShardingSphere 也支持数据库属性的动态变更，我们基于自己的内部系统的特征，实现了基于Consul的资源变根通知。我们的资源存在Consul。\n\nArena-Jdbc 支持对使用的资源做无损的变更，Arena-Jdbc 收到资源变更时，会先对新下发的资源做预热处理，预热后，再切换使用的数据源，切换成功后，再销毁老的数据源，业务无感知。\n\n如果新的资源预热失败，则不会做变更处理，保证下发的资源时可用的，规避错误下发的问题。\n\n扩容和缩容也是同理，一期数据需要运维手动迁移，迁移好了后，直接在PaaS平台下发新的配置即可，二期支持自动迁移数据和配置变更结合。\n\n同时支持Proxy的无损上下线机制，通过PaaS平台对Proxy的变更，把需要下线的Proxy节点去掉，通知 Arena-Jdbc，Arena-Jdbc 会把缩容的Proxy节点去掉，做到无损下线。\n\n \n\n# **读写分离**\n\n \n\n读写分离我们完全基于Apache ShardingSphere的来实现，我们根据喜马拉雅业务的特性，对强制路由做了增强，不需要规则配置为 Hint 模式，只要线程上下文带有强制路由的标志，就可以路由到指定的库和表，不受分表规则的影响，我们重写了ShardingStandardRoutingEngine的Sharding时路由库和表的逻辑：\n\n```java\nprivate Collection<String> routeDataSources(final TableRule tableRule, final ShardingStrategy databaseShardingStrategy, final List<ShardingConditionValue> databaseShardingValues) {\n        //先判断是否存在Hint上下文路由标，如果有,则优先根据用户指定的规则路由库\n        Collection<Comparable<?>> databaseShardings = HintManager.getDatabaseShardingValues(tableRule.getLogicTable());\n        if (databaseShardings != null && databaseShardings.size() > 0) {\n            List<String> list = new ArrayList<>(4);\n            for (Comparable<?> databaseSharding : databaseShardings) {\n                list.add((String) databaseSharding);\n            }\n            if (log.isDebugEnabled()) {\n                log.debug(\"route dataSources, find HintManager, so hint to: {}\", list);\n            }\n            return list;\n        }\n        //没有Hint路由规则，则按Sharding 规则路由\n        if (databaseShardingValues.isEmpty()) {\n            return tableRule.getActualDatasourceNames();\n        }\n        Collection<String> result = new LinkedHashSet<>(databaseShardingStrategy.doSharding(tableRule.getActualDatasourceNames(), databaseShardingValues, properties));\n        Preconditions.checkState(!result.isEmpty(), \"no database route info\");\n        Preconditions.checkState(tableRule.getActualDatasourceNames().containsAll(result), \n                \"Some routed data sources do not belong to configured data sources. routed data sources: `%s`, configured data sources: `%s`\", result, tableRule.getActualDatasourceNames());\n        return result;\n    }\n```\n\n \n\n路由表也是同样的逻辑，我们重写了ShardingStandardRoutingEngine的routeTables方法，和上面一样。先从Hint的上下文获取。这样通过上下文的方式能很好的满足业务个性化的路由规则，能和Sharding规则共存。\n\n \n\n# **Database Plus**\n\n \n\nApache ShardingSphere 除了提供基本的分库分表，读写分离的能力外，在上层还提供了很多的插件和扩展的机制，这让我们在基于数据库提供更偏向业务的起的能力非常容易，成本非常低,这叫 Database Plus\n\nDatabase Plus 简单的说就是你用 Apache ShardingSphere 的数据库中间件，不仅仅是提供了分库分表这一基本能力，通过对底层数据的封装为统一的交互标准插件模式，可以在上面实现很多业务的通用的场景的需求，比如喜马拉雅除了用到 Apache ShardingSphere 基础的能力外，我们也享受了 Database Plus 的威力，我们在它的基础上轻松实现了支持压测的影子库和影子表，数据加解密，机房级别容灾的同城双读，分布式唯一ID。\n\n \n\n# **影子库和影子表**\n\n \n\n影子库影子表我们对Apache ShardingSphere做了改动，Apache ShardingSphere需要修改sql，我们认为对业务有改造成本，同时结合我们自己的压测平台，我们和业界一样，我们也实现了影子标记，通过全链路压测标的传递来判断是否路由到影子库/影子表，业务无需任何改造，即可使用影子库影子表来做压测，同时不需要在运行时对sql改写，提升了性能，我们重写了ShadowSQLRouter,\n\n```java\npublic class ArenaShadowSQLRouter extends ShadowSQLRouter {\n    @Override\n    public boolean isShadow(final SQLStatementContext<?> sqlStatementContext, final List<Object> parameters, final ShadowRule rule) {\n        if (sqlStatementContext instanceof InsertStatementContext || sqlStatementContext instanceof WhereAvailable\n            || sqlStatementContext instanceof UpdateStatementContext) {\n                 //这里就是判断是否有压测标，如果有，ShardingSphere则会找影子的逻辑。\n                return ArenaUtilities.checkPeakRequest();\n        }\n        return false;\n    }\n}\n```\n\n \n\n通过spi的方式把我们自定义的ArenaShadowSQLRouter给Apache ShardingSphere加载使用，不得不说Apache ShardingSphere的插件设计很赞，很方便自定义和扩展。\n\n配置还是和Apache ShardingSphere的一样：\n\n```yaml\n# 配置影子库规则\n- !SHADOW\n    # true-影子表，false-影子库(默认)\n    enableShadowTable: true \n\n    # 源库名称(对应DataSources数据源配置中的名称)，影子库才需要配，影子表不需要配置\n    sourceDataSourceNames:  \n      - ds0           # 源库，与影子库shadow_ds0对应\n      - ds1           # 源库，与影子库shadow_ds1对应\n\n    # 影子库名称(对应dataSources数据源配置中的名称)，影子库才需要配，影子表不需要配置\n    shadowDataSourceNames:  \n      - shadow_ds0    # 影子库，与源库ds0对应\n      - shadow_ds1    # 影子库，与源库ds1对应\n```\n\n \n\nenableShadowTable 我们新增了该属性，来确定是使用影子库还是影子表。\n\n \n\n# **影子库**\n\n \n\n影子库，一定要填sourceDataSourceNames和shadowDataSourceNames，enableShadowTable不用设置，或者设置为false。\n\n \n\nsourceDataSourceNames\n\n按顺序映射，一一对应：\n\nds --> shadow_ds\n\nds1--> shadow_ds1\n\n影子库/影子表是最后一个路由规则，如果发现有影子库/影子表，则根据实际的库找到对应的影子库/影子表，执行sql\n\n \n\n# **同城多活**\n\n \n\n基于喜马拉雅的业务特性，读多写少，我们只实现了对读业务的双机房部署，写业务还是路由到主机房。\n\n为了增强容灾能力，喜马拉雅搭建了双机房，同时承载业务流量，当一个机房故障时，可以把流量快速切换到另一个机房，我们在dal层设计上支持双写，这里充分利用了 Apache ShardingSphere 的读写分离功能，读和写可以配置独立的数据源，我们只需要在上面做了一层封装，在切换时候动态变更对应的数据源即可，为了切换时不影响业务的流量，我们是先预热新的数据源，再销毁老的数据源。\n\n架构图如下：\n\n \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ODgzZmMyZDliZDcwMTViYTIzYmE5MGE1MWE4YWJjNjBfQVU2NXhzWkIxTzdwZTJqT3MxNVNuRmdENUNqUExhRHNfVG9rZW46Ym94Y250TkVLaXJ1dzJZSlNnTDdlMkJOWk9lXzE2ODAwNzcwNDk6MTY4MDA4MDY0OV9WNA)\n\n \n\n另外我们也对双写做了研究和探索，关键在于数据库的双向同步，基于阿里开源的 otter 做了改造，支持基于 gtid 模式同步，不依赖打标，打标回有性能开销，在一些业务做了试用。\n\n \n\n# **分布式唯一ID**\n\n \n\n分库分表后，唯一id是必须要满足的需求，Apache ShardingSphere默认提供了snowfake和uuid算法，但不是很时候db的场景，db需要保证顺序和格式，所以我们基于Apache ShardingSphere提供的接口，也实现自己的唯一id生成策略：数据分片后，不同Mysql实例生成全局唯一主键是非常棘手的问题。Arena-Jdbc 实现了Apache ShardingSphere的分布式主键生成器接口，通过集成喜马拉雅内部的全局唯一id生成服务，提供了适用于喜马拉雅内部的自增主键生成算法-BoushId主键生成策略。\n\n \n\n# **监控和报警**\n\n \n\n做一个数据库中间件，监控是必不可少的部分，就像我们的眼睛，没有监控就是瞎抓，以及对异常情况的报警也是非常重要的部分，只有完善的监控和报警才能算是一个完整的产品，得益于 Apache ShardingSphere 在设计时就提供了钩子，我们能非常小的成本就能实现对sql层面的监控和报警。\n\n \n\nArena-Jdbc 客户端通过钩子回调，从多维度数据来分析使用数据库的运行情况，以30s为一次统计周期，每个周期统计的数据包括：Mysql总请求量，新增、删除、修改和查询的请求量，失败的请求量和慢请求量，影子库的流量，以及统计响应时间的TP百分比，还有连接池的等待时间、建连时间、连接数等信息。这些指标会发送给专门的收集指标服务，并持久化到时序数据库，PaaS平台可以从时序数据库中查询数据，展示给各个业务，对于异常sql和慢sql，做报警等后续处理。\n\n \n\n# **其他**\n\n \n\n我们除了基于Apache ShardingSphere实现上述关键特性外，我们还对Apache ShardingSphere做了一些优化和改进，以更适合喜马拉雅的业务。\n\n \n\n· 优化分片规则，启动时，如果分片的真实表不存在的情况则报错，将配置错误前置\n\n \n\n· 有的业务方有几百，甚至几千的分表，这种情况下，由于Apache ShardingSphere中的联邦查询需要依次扫表，启动速度很慢，达到了分钟级别。针对这种情况，我们新增了props配置项，不再初始化联邦查询，打打加快了启动速度，并且再使用中，也没有用联邦查询\n\n \n\n· 优化了Apache ShardingSphere，执行sql异常不报错误的情况\n\n \n\n· 由于有的业务方，对重要的表采用了大写的表名和列名，我们去掉Apache ShardingSphere中，对配置中的大写的表名列名强制小写的情况，允许大写的表名和列名\n\n \n\n· 新增了props配置项，可以调节Apache ShardingSphere的编译缓存的大小\n\n \n\n· 优化Apache ShardingSphere复合分片算法，精确匹配分片字段\n\n \n\n在ComplexShardingStrategyConfiguration中，添加shardingColumnList字段：修复Apache ShardingSphere，批量insert，不返回主键的问题，这个问题在mybatis-plus中比较常见\n\n \n\n· 不分片的表，支持使用默认的主键id生成策略\n\n \n\n# **总结**\n\n \n\n基于Apache ShardingSphere实现的数据库中间件Arena-Jdbc，经过半年的时间，已经覆盖了喜马拉雅的70%的核心业务，目前没有发现任何问题，表现的非常稳定，通过和我们的PaaS平台结合，业务也非常愿意接入，另外我们使用Apache ShardingSphere时，社区还没有发布stable的版本，所以我们在使用过程中也遇到了些问题，基本上我们都解决了，有的社区也有对应的解决方案，得益于社区非常活跃，我们以后也希望把我们做的一些feature能回馈到社区，为Apache ShardingSphere的发展做出一点点小贡献。\n\n非常感谢基础架构团队胡建华、彭荣新提出的宝贵意见和建议，感谢喜马拉雅基础小伙伴们在项目推广过程中的大力支持，让Apache ShardingSphere在喜马拉雅生根发芽。\n\n非常感谢亮哥亲自来喜马拉雅对Apache ShardingSphere的技术内幕和规则做了一次全面的分享，非常关心我们在使用Apache ShardingSphere过程中遇到的问题，在现场对小伙伴提的问题都一一作了 深入的解答，非常感谢亮哥，祝Apache ShardingSphere越来越好。","date":"2022-05-11","author":"沈辉  基础架构","excerpt":"基于Apache ShardingSphere实现的数据库中间件Arena-Jdbc，经过半年的时间，已经覆盖了喜马拉雅的70%的核心业务，目前没有发现任何问题，表现的非常稳定，通过和我们的PaaS平台结合，业务也非常愿意接入","metaDescription":null,"createdAt":"2023-03-29T08:09:25.219Z","updatedAt":"2023-03-29T08:09:26.226Z","publishedAt":"2023-03-29T08:09:26.216Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":13,"attributes":{"feature":true,"title":"ShardingSphere 助力当当 WMS：订单效率提升 30%、节约成本上千万","content":">Apache ShardingSphere 简单易用且成熟稳定，当当自研 WMS 如虎添翼，在供应链系统改革中起到了至关重要的作用，是我们的杀手锏。\n——李勇，当当 WMS 技术负责人\n\n>Apache ShardingSphere 5.0.0 GA 已于 2021 年 11 月发布，5.1.0 版本也于上周发布。历经两年的迭代与打磨，可插拔生态趋近成熟，标志着 Apache ShardingSphere 从分库分表中间件蜕变为以 Database Plus 为理念的分布式数据库生态系统。\n\n近年来，随着“建设社会主义文化强国”政策的持续落实，全民阅读在移动互联快速发展中得到大力倡导，阅读给当代人提供了丰富的精神食粮。当当成立于 1999 年底，作为新技术与传统行业的结合体，当当融合了互联网新技术和传统图书行业，二十余年深耕图书领域助力全民阅读。\n\n当当在刚成立的几年，也是中国互联网腾飞的阶段，电商逐渐多元化发展的趋势迎来了行业野蛮生长期，各电商平台的发展如逆水行舟，不进则退。当当对战略和管理等方面进行调整的同时，也对技术架构层进行了硬核升级。\n\n当当成立之初的仓储物流系统并非自研，随着业务体量、技术实力等多方面的变化，为了满足业务快速发展需要，当当决定重构 WMS（Warehouse Management System）和 TMS（Transportation Management System）。硬件层使用 x86 取代小机，数据层由传统集中式转型为更灵活的分布式架构。如何解决海量仓储货品信息问题，是重构待解决的核心问题之一。互联网技术栈比较常用的分库分表被率先尝试使用，但由于当时开源数据库中间件的成熟度及功能方面不满足预期，当当决定自研分库分表产品，Sharding-JDBC 就这样应运而生，为“连接”带来更多可能。\n\n**自研 WMS 的上线完成了仓储层智能改革，五年、十余仓，Apache ShardingSphere 同 WMS 完美支撑了数个“读书节”、“618”和“11.11”等大型促销活动。**\n\n\n## 业务挑战\n\n当当在自研 WMS 之前使用的是第三方仓储管理系统，其数据库是基于 IBM 小机的 Oracle，结合当当业务体量以及电商仓储物流在促销期间密集请求的业务特点，传统集中式数据库架构给 WMS 带来了极大的挑战。\n\n在业务快速发展的冲击下，数据库计算和存储能力的上限越发明显，Scale-up 方案难以满足超级促销期间的业务需求，技术团队需要对业务层多次做拆分调整来缓解“存算”瓶颈，产能存在受限的风险。\n\n- 计算、存储能力双受限\n\n传统集中式架构扩展能力相对较差，数据库的计算和存储能力成为明显瓶颈；\n\n- 业务开发维护成本高昂\n\n在扩展能力相对差的基础上，业务侧需要不断的去让步做拆分，开发和维护成本持续攀升；\n\n- 系统架构开放度不足\n\n系统架构开放度不足将会使系统面临灵活度欠佳、功能缺失以及转型难度高等问题，基于现有架构难以快速响应新业务的开展及相关政策的调整，如云原生、SQL 审计、数据脱敏以及分布式治理等多方面需求。\n\n\n## 解决方案\n\n当当技术部结合现状及未来发展等多维度的调研，最终确定了自研 WMS 的方案。在硬件方面全面采用通用的 x86 设备替换 IBM 小型机，数据库层面引入开源数据库 MySQL 来替换 Oracle。当时已有多个开源数据库中间件，但成熟度和功能方面与预期不符，因此选择了自研数据库中间件，Sharding-JDBC 闪亮登场。\n\nShardingSphere-JDBC 定位为轻量级 Java 框架，在 Java 的 JDBC 层提供额外服务。具备轻量、高效、易用以及高兼容等特点。它使用客户端直连数据库，以 jar 包形式提供服务，无需额外部署和依赖，可理解为增强版的 JDBC 驱动，完全兼容 JDBC 和各种 ORM 框架。\n\n- 适用于任何基于 JDBC 的 ORM 框架，如：JPA、Hibernate、Mybatis、Spring JDBC Template 或直接使用 JDBC；     \n\n- 支持任何第三方的数据库连接池，如：DBCP、C3P0、BoneCP、HikariCP 等；\n\n- 支持任意实现 JDBC 规范的数据库，目前支持 MySQL、PostgreSQL、Oracle、SQL Server 以及任何可使用 JDBC 访问的数据库。\n\n![当当1.png](https://official-media.sphere-ex.com/1_16c3bbe14b.png)\n\nShardingSphere-JDBC 方案\n\n目前 Apache ShardingSphere 由 JDBC、Proxy 和 Sidecar（规划中）这 3 款既能够独立部署，又支持混合部署配合使用的产品组成，在当当自研 WMS 中使用的是 ShardingSphere-JDBC。\n\n那么 ShardingSphere-JDBC 在 WMS 中具体是如何应用的呢？\n\n在 WMS 系统中，城市的“仓”就是仓储系统的管理单位，每个仓有对应的业务系统和数据库，单个仓使用 3 套 MySQL 主从集群承载指定城市仓库的数据。目前，当当在全国各地设有十余个自建的仓库，主要分布在订单量较大的城市，这种自建仓库模式在具备灵活管理等特性的同时，也可以降低企业的长期仓储成本。\n\n在架构规划上，WMS 通过 ShardingSphere-JDBC 按业务类型进行分库， 每套集群负责存储指定的业务数据，单个仓的 3 套 MySQL 集群分工如下：\n\n- 基础库集群：User，Area，Menu；\n\n- 业务库集群：Order，Package；\n\n- 库存库集群：Stock，Working。\n\n上线前，系统会根据仓库的基础数据进行初始化，如货位等信息，这里不做深入讨论。\n\n分布式数据库中间件的架构解决了 WMS 存算受限、开发维护成本高以及系统灵活度不足的问题，以成熟的底座和熟悉的技术栈去承载业务，架构相对透明化，兼容度良好且没有入侵性。\n\n![当当2.png](https://official-media.sphere-ex.com/2_783843d774.png)\n\n当当 WMS 系统拓扑图（单仓）\n\n\n## 用户收益\n\n在当当自研 WMS 的过程中发挥了非常重要的作用，带来了五方面的收益如下：\n\n1、性能极致\n\nShardingSphere-JDBC 轻量级的框架使其性能接近原生 JDBC，提供分库能力的同时性能可发挥极致，WMS 产能不再有受限的风险；\n\n2、系统稳定\n\n自 2016 年 WMS 上线至今，数年来系统一直稳定运行；\n\n3、低风险、零侵入\n\n当当底层系统为 2000 年演进而来，Apache ShardingSphere 凭借自身零侵入性的特点，用很小的改动就即完成了兼容适配，满足了业务需求；\n\n4、专注业务开发\n\n研发团队无需再关心业务拆分的工作，可将更多的精力聚焦于业务需求；\n\n5、降本增效\n\n高兼容的特点可以在无需大量的重构和改造前提下满足业务增长的需求，改造迁移成本低；     \n\n通过仓店智能化、入库商品自动匹配最优货位的设计，仓店订单效率提升了 30%，人力成本节约上千万。\n\n\n## Apache ShardingSphere 的前世今生\n\n有人说 ShardingSphere 是当当的产品，其实更准确的说：ShardingSphere 起源于当当，在 2018 年 11 月 10 日捐赠给 Apache，经 17 个月的孵化于 2020 年 4 月 15 毕业，是 2020 年度第一个从 Apache 孵化器毕业的顶级项目。\n\n前段时间，在 ShardingSphere 进入 Apache 基金会三周年的纪念日（2021/11/10）里，社区发布了期待已久的 5.0 正式版，借着这个机会，一起回顾 Apache ShardingSphere 的前世今生。\n\n- 2014 年，当当针对电商平台推出了统一开发框架 dd-frame，目的是通过分离业务和技术代码，实现开发框架统一化、技术组件标准化以及跨组沟通高效化，使研发人员的精力最大化的聚焦在业务开发上。开发框架中名为 dd-rdb 的关系型数据库模块专门用于负责数据访问，可实现简单的分库分表功能，它就是 Sharding-JDBC 的前身，也是 dd-frame 2.x 中重点规划的部分。     \n \n- 2015 年，当当决定重构 WMS 和 TMS，设计上需要引用分库分表方案，天时地利人和，当当架构部在 9 月正式启动了项目，同年 12 月份 Sharding-JDBC 1.0.0 在当当内部发布，正式开启了 ShardingSphere 的征途。\n \n- 2016 年初，Sharding-JDBC 从 dd-rdb 中剥离，向社区开源，产品形态是增强版 JDBC 驱动，以 jar 包形式提供服务。\n \n- 2017 年底，发布 2.0.0 版本，打开了数据库治理功能。\n\n- 2018 年，进入 Apache 软件基金会。3.0.0 版本是一个华丽的转身，独立服务形态的 Sharding-Proxy 面世，实现了异构开发语言的需求，同时项目名称从 Sharding-JDBC 更名为 ShardingSphere，迈出了构建数据库上层标准和生态的第一步。     \n \n- 2019 年，发布 4.0.0 版本，可支持更多的数据库产品。\n \n- 2020 年，毕业成为 Apache 软件基金会的顶级项目。    \n\n- 2021 年 11 月 10 日，发布 5.0.0 GA 版，作为加入 Apache 软件基金会三周年纪念日的礼物，呈现给社区及整个分布式数据库和安全生态领域。     \n\n![当当3.png](https://official-media.sphere-ex.com/3_ae2b522cd5.png)\n Apache ShardingSphere - RoadMap\n\n从 5.0.0 版本开始，Apache ShardingSphere 正式开启了以可插拔架构为核心的全新阶段，实现了由过去数据分片单一应用场景到现在复杂应用环境下综合数据治理的身份转变与能力提升。在新版本中，Apache ShardingSphere 具备了更多新特性，并拥有了更加成熟的大数据解决方案。\n\n首先，基于可插拔架构对内核进行了全面改造，内核中的各个功能可以任意组合并叠加使用。其次，为了提升 SQL 分布式查询能力，5.0.0 GA 版打造了全新的 Federation 执行引擎，来满足用户复杂的业务场景。此外，5.0.0 GA 版在内核功能 API 层面也进行了大量优化，旨在降低用户使用这些功能的成本。\n\n## 写在结尾\n\n全链条数字化驱动着当当高质量发展，让全民阅读触手可及。在必不可少的自研 WMS 一环中，ShardingSphere 在数据层提供了强力支撑。\n\nApache ShardingSphere 历经两年的迭代与打磨，5.0.0 GA 版本已面世。趋近成熟的可插拔生态，标志着 Apache ShardingSphere 从分库分表中间件工具蜕变为以 Database Plus 理念为新航标。重塑分布式可插拔体系，助力整个数据库行业带来新风向和更有价值的解决方案。\n\n更多内容欢迎点击阅读原文链接访问中文社区 OpenSEC（Open SphereEx Community），一起畅聊开源。\n\n","date":"2022-04-29","author":"SphereEx","excerpt":"历经两年的迭代与打磨，可插拔生态趋近成熟，标志着 Apache ShardingSphere 从分库分表中间件蜕变为以 Database Plus 为理念的分布式数据库生态系统。","metaDescription":null,"createdAt":"2022-04-29T08:49:24.970Z","updatedAt":"2022-11-02T05:36:20.351Z","publishedAt":"2022-04-29T08:49:28.304Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[{"id":14,"attributes":{"feature":false,"title":"Asia’s E-Commerce Giant Dangdang Increases Order Processing Speed by 30% — Saves Over Ten Million in Technology Budget with  Apache ShardingSphere","content":">Apache ShardingSphere is an easy-to-use and stable product, making Dangdang’s warehouse management system (WMS) even more powerful. Integrated with WMS, ShardingSphere plays a vital role in reforming the supply chain system.\n>-- Li Yong, Head of WMS Technology, Dangdang\n\nFollowing Apache ShardingSphere 5.0.0 GA release in November 2021, the 5.1.0 version was released last month. Having gone through over two years of polishing, ShardingSphere’s plugin-oriented ecosystem is beginning to take shape, and the project embarks on the evolution from a simple data sharding middleware to a mature distributed database ecosystem driven by the concept of Database Plus.\n\nDangdang, established at the end of 1999, has become a leading e-commerce platform selling books of any kind, and by integrating new Internet technologies with the traditional book industry. Dangdang was founded during the surge in China’s Internet industry in the early 2000s.\n\nLater, the e-commerce industry became exremely competitive in the country, and the market saturated. Facing fierce market competition, e-commerce platforms had to adapt to remain competitive.\n\nIn response, Dangdang not only adjusted its business strategies and management approaches, but also upgraded its technology architecture. Dangdang didn’t have its warehouse management and transportation system at that time. However, with growing business volume and technological capabilities, Dangdang needed to rebuild its warehouse management system and transportation management system (TMS) to better satisfy its business needs. For instance, in terms of hardware, it replaced mini-computer with x86, while its old centralized system was transformed to a distributed system with more flexibility.\n\nOne of the biggest challenges was massive warehousing data storage. The engineers wanted to adopt the data sharding technology that was often chosen by other big Internet companies. Disappointingly, they failed to find a mature and versatile open source database middleware in the marketplace, and therefore,started to develop a new data sharding product. That’s the origin of Sharding-JDBC. The product was created to bring more possibilities to data services.\n\nDangdang released its new WMS five years ago, which meant that it completed its intelligent warehousing transformation. Since then, with Apache ShardingSphere, the WMS has enabled Dangdang to hold large online shopping events every year such as Dangdang’s April Reading Festival, Double-Eleven Online Shopping Festival (aka Singles Day), and Mid-Year Shopping Festival, and to manage over a dozen of smart warehouses.\n\n## Business Challenges\n\nWhen Dangdang adopted a third-party WMS, the database in use was Oracle based on IBM minicomputers.\n\nHowever, considering the increasing business volume and warehouse order requests, especially during the online shopping festivals, the traditional centralized database architecture of the old WMS became inadequate because the computing and storage capabilities were limited. Additionally, the scale-up solution couldn’t support the system during online shopping festivals, and therefore the developers must do scale-up and adjust the business layer several times to alleviate the storage and computing limits and avoid production risks.\n\n- **Limited computing and storage capabilities**\n\nA centralized architecture is less scalable, making database computing and storage capabilities become the bottleneck.\n\n- **Expensive development and maintenance cost**\n\nBecause of poor scalability, developers have to make concessions and scale-up, which increases system development and maintenance costs.\n\n- **Exclusiveness**\n\nIf the architecture is not open enough, the system is less flexible, with fewer functions, and difficult to be transformed. The current architecture makes it difficult to quickly adopt new business services such as cloud native ones, SQL audit, data encryption, and distributed governance.\n\n## The Solution\n\nBased on the situation described above, Dangdang’s tech team proposed a warehouse management system solution: in terms of hardware, IBM minicomputer would be replaced with all-purpose x86, and MySQL would replace Oracle.\n\nHowever, at that time, there wasn’t a versatile and mature enough open source database middleware living up to Dangdang’s expectations, so they created one and named it Sharding-JDBC.\n\nShardingSphere-JDBC is positioned as a lightweight Java framework that provides additional services at the Java Database Connectivity (JDBC) layer. It is lightweight, efficient, easy to use, and compatible.\n\nWith ShardingSphere providing services in the form of the .jar files package, users can connect the client directly to the database without additional deployment and dependencies. It can be seen as an enhanced JDBC driver, fully compatible with JDBC and all ORM frameworks.\n\n- Compatible with JDBC and any JDBC-based ORM framework such as JPA, Hibernate, Mybatis, Spring JDBC Template.\n\n- Supports all third-party database connection pools such as DBCP, C3P0, BoneCP, HikariCP, etc.\n\n- Supports all databases implementing JDBC standards. Currently, ShardingSphere-JDBC supports MySQL, PostgreSQL, Oracle, SQL Server, and any database that can be accessed via JDBC.\n\n![1.jpeg](https://official-media.sphere-ex.com/1_3d03cbca69.jpeg)\nThe ShardingSphere-JDBC Topography\n\nCurrently, Apache ShardingSphere consists of three products, i.e. JDBC, Proxy, and Sidecar (TODO). ShardingSphere-JDBC and ShardingSphere-Proxy can be deployed independently or together.\n\nIt is ShardingSphere-JDBC that is used in Dangdang’s warehouse management system.\n\n\n>So how is ShardingSphere-JDBC exactly utilized?\n\nIn the warehouse management system, each warehouse positioned in a physical city is referred to as a unit with its corresponding business system and databases. Each warehouse has three sets of MySQL primary-secondary clusters to load the warehousing data of the designated city. So far, Dangdang has more than ten self-built warehouses all over China, mostly in cities where customers place a large number of orders. This self-built warehouse model is flexible for warehouse management and helps reduce storage costs in the long run.\n\nIn terms of architecture, the WMS uses ShardingSphere-JDBC to do database sharding according to their business types, and each cluster stores specified business data. The three MySQL clusters of a single warehouse are divided into three types as follows:\n\n-** Basic:** stores user, area, and menu data.\n- **Business:** stores order and package data.\n- **Inventory:** stores stock and working data.\n\nBefore the release, the system was initialized based on the basic data of a warehouse, such as storage locations.\n\nNext, by deploying the distributed database middleware, Dangdang successfully solved a series of problems such as limited storage and computing capabilities, high costs, and lack of flexibility.\n\n![1.jpeg](https://official-media.sphere-ex.com/1_b7199ae49f.jpeg)\nTopography of Dangdang’s Warehouse Management System (of a Single Warehouse)\n\n## User Advantages\n\nApache ShardingSphere played a significant role in helping Dangdang develop its WMS. There are five main benefits:\n\n- **Extraordinary performance**\n\nThe ShardingSphere-JDBC lightweight framework makes its performance close to a native JDBC. Apart from its great database sharding capability, it helps database performance be taken to the extreme, which allows the WMS to work at full capacity.\n\n- **Keep the system stable**\n\nWMS has been functioning well since its release in 2016.\n\n- **Low risk & zero invasion**\n\nThe underlying system of Dangdang has been evolving since 2000. Thanks to its zero-intrusion nature, Apache ShardingSphere can be compatible with others with small modifications to meet Dangdang’s business requirements.\n\n- **Allow developers to focus on the business side**\n\nThe developer team does not need to worry about sharding any more and can concentrate on developing the system to meet business needs.\n\n- **Cost effective and efficient**\n\nSince ShardingSphere is known for its high compatibility, to satisfy increasing business needs, developers don’t have to reconstruct or upgrade the system, minimizing the migration cost.\n\nWarehousing order processing speed is increased by 30% and accordingly, tens of millions of manpower costs are reduced due to the smart warehouses and the auto storage location matching technology.\n\n**Apache ShardingSphere’s Roadmap**\n\nSome said that ShardingSphere is a product created by Dangdang. To be precise, ShardingSphere was derived from the company and it also donated ShardingSphere to the Apache Software Foundation (ASF) on November 10, 2018.\n\nAfter 17-months in the ASF incubator, Apache ShardingSphere successfully graduated on April 15, 2020, as a Top-Level Apache Project.\n\nRecently, to celebrate the third anniversary of ShardingSphere entering Apache Software Foundation(ASF), the community released ShardingSphere 5.0.0. Below is a brief review of Apache ShardingSphere.\n\n- In 2014, Dangdang introduced a centralized development framework targeting at its e-commerce platform called dd-frame. It was created to unify the development framework, standardize its technical components, and achieve efficient cross-team communication by separating business code from technical code. In this way, engineers can devote all their efforts to the business side. The relational database module named dd-rdb in the framework was developed to handle data access and implement the data sharding function. It was the precursor of Sharding-JDBC, as well as a major part of dd-frame 2.x.\n\n- In 2015, Dangdang decided to rebuild its WMS and TMS. As it needed a data sharding plan, the team launched the project in September. In December, 2015, Sharding-JDBC 1.0.0 was released and used within Dangdang.\n\n- In early 2016, Sharding-JDBC was separated from dd-rdb and became open source. The product is an enhanced JDBC driver providing service in .jar files.\n\n- At the end of 2017, Version 2.0.0 was released with the new data governance function.\n\n- In 2018, ShardingSphere was enrolled into Apache Incubator. The release of Version 3.0.0 was a notable turnaround: Sharding-Proxy was released as an independent service. It supported heterogeneous languages, and the project was renamed from Sharding-JDBC to ShardingSphere. It’s in 2018 that the community decided to build the criteria and ecosystem above databases.\n\n- In 2019, Version 4.0.0 was released capable of supporting more database products.\n\n- In 2020, ShardingSphere graduated as a Top-Level Project of the ASF.\n\n- On November 10, 2021, Version 5.0.0 GA was released as a third-anniversity celebration with the whole Apache ShardingSphere community, and the distributed database industry.\n\n![1.jpeg](https://official-media.sphere-ex.com/1_83a06f3e37.jpeg)\nApache ShardingSphere—Roadmap\n\nSince Version 5.0.0, Apache ShardingSphere has embarked on its new journey: with the plugin oriented architect at its core, it evloved from a data sharding application to a comprehensive and enhanced data governance tool applicable to various complex application scenarios. Concurrently, Apache ShardingSphere also has more features, and big data solutions.\n\n## Conclusion\n\nDigitization motivated Dangdang to achieve high-quality development and fulfill its mission. ShardingSphere is glad to support Dangdang’s WMS with its cutting-edging data services.\n\nHaving gone through two years‘ development, Apache ShardingSphere 5.0.0 GA has been released. The pluggable ecosystem marks an evolution from a data sharding middleware tool to a pioneer in the industry following the “Database Plus” concept.","date":"2022-03-11","author":"SphereEx","excerpt":"Apache ShardingSphere is easy-to-use and stable, making Dangdang’s warehouse management system (WMS) even more powerful.","metaDescription":"Dangdang is a B2C-based e-commerce company in China offering a large selection of products both online and offline.","createdAt":"2022-04-29T10:52:10.330Z","updatedAt":"2022-09-28T02:42:43.896Z","publishedAt":"2022-04-29T10:52:12.009Z","locale":"en"}}]}}},{"id":40,"attributes":{"feature":true,"title":"基于Sharding-JDBC的订单分库⽅案","content":"# **前言** \n\n随着业务的发展，数据量迅速增⻓，单实例 Mysql 数据库的磁盘存储容量很快就会达到上限（阿⾥云 RDB 最⼤上限6T）。因此为了解决⽣产环境的存储瓶颈，需要对数据库进⾏分库设计，加⼤存储空间。**本⽅案基于Apache开源项⽬ ShardingSphere 采 Sharding-JDBC 进⾏订单⽔平分库设计（数据分⽚）缓解存储压⼒。**\n\n \n\n# **方案对比**\n\n⽬前主流的分库分表⽅案就是 Sharding-JDBC 和 Mycat，下⾯主要对⽐这两种。\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MGZlMTlmM2IyYTNiZDYzMTk5YjI0NWVlYjJlMzE2NmZfRXVzMURvRUdabmYxR1J4MTJxVWFkRGNuVE1GbjJia21fVG9rZW46Ym94Y25EMDBYVHBVUFhuRm51ZVpvS2Rja3ZjXzE2ODAwNzQ5MzU6MTY4MDA3ODUzNV9WNA)\n\n经上述对⽐，可以发现 Sharding-JDBC 是直连数据库、没有中间层的⽅式，性能上更优，同时由于 Sharding-JDBC 集成在应⽤代码内，并不会增加额外的运维成本，使开发者可以专注于⾃⾝代码逻辑。另外，Sharding-JDBC ⽬前仅⽀持 Java，考虑到⽬前团队技术栈主要以 Java 为主，所以采⽤ Sharding-JDBC 的分库⽅案成为了⼀个不错的选择。 下图反映了 Sharding-JDBC 在应⽤中所处的位置：\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=NTY1M2UyZjBmNWE1NThiMzg1YmI0YTczYmNmNDZhMjZfdHJtWU1pSUdNVnRLV3JabHo0cktNUndXbGxEWkdXYXhfVG9rZW46Ym94Y25XVGdhUGdOVFVid21ucHlTWnpFSFNkXzE2ODAwNzQ5MzU6MTY4MDA3ODUzNV9WNA)\n\n \n\n# **原理**\n\nSharding-JDBC 提供了精确分⽚扩展接⼝ org.apache.shardingsphere.api.sharding.standard.PreciseShardingAlgorithm. PreciseShardingAlgorithm，**只需要实现其doSharding ⽅法，基于分⽚键做⼀定的算法转换，返回指定的数据源，便可以达到分库的效果。**  ⽬前我们使⽤的是⼀致性 hash 算法，⽤于定位分⽚键所在数据源，同时引⼊虚拟节点，可以避免数据倾斜问题（即有的数据源数据多，有的数据源数据少，导致数据源设备压⼒不均衡）算法原理如图： \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=NWZkMDc1YmUxZTFmMzk5ZTg2ZThlYzhhM2NkZGU5MTRfa2ZzNDdsR3FTbnhJVTJaTENmRGFHMkZBMUN2S3hhN0pfVG9rZW46Ym94Y25qUWtZdWRYTmR0VG9SUHdtcTF5SFVnXzE2ODAwNzQ5MzU6MTY4MDA3ODUzNV9WNA)\n\n **然后基于该分库的分⽚键&分⽚算法，我们还需在 application.yml 中增加 sharding 数据源配置；同时需要增加分表相关的配置信息。**最后项⽬启动时，sharding 会⾃动加载这些配置初始化数据源。在请求时从请求参数中提取到分⽚键，应⽤分⽚算法，找到⽬标数据源，操作该数据源。 \n\n# **实战**\n\n在原有订单数据源 datasource_name 的基础上，新加3台订单数据库实例 datasource_name_1&datasource_name_2&datasource_name_3，扩充后共4台订单数据库实例，磁盘存储最⼤可以达到原有容量的4倍。\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=NjUzZGZlZDA4MmIwNWE4MTRhN2Y1MWIyZWFkNDc3NTVfcUZRMWFIMFNQWnNxR0UxTUVmc21uSmttRElTMjNETnBfVG9rZW46Ym94Y25GZmVJWWphbTJBY3lTOTdCSkZKNURnXzE2ODAwNzQ5MzU6MTY4MDA3ODUzNV9WNA)\n\n**4.1引⼊maven依赖**\n\n```xml\n<dependency>\n\t<groupId>org.apache.shardingsphere</groupId>\n\t<artifactId>sharding-jdbc-spring-boot-starter</artifactId>\n\t<version>4.1.1.001</version>\n</dependency>\n```\n\n这⾥的4.1.1.001版本是在官⽹ sharding 4.1.1版本上⾃⾏扩充了⼀部分内容：本⾝根据业务分⽚键对业务数据进⾏分⽚后，同⼀类型的数据只会在⼀个数据源中，sharding 只能操作这⼀个数据源的数据，但对于⼀部分分⽚键落⼊新数据源后，意味着数据源发⽣了变动，⽽在旧库数据未迁⼊到新库时，会存在⼀定的兼容时期，此时，改造源码实现了如下双操作（除插⼊外）：a、双查询：查询新旧数据源的数据，然后业务项⽬⾥使⽤切⾯进⾏查询汇总去重，根据 id+updateDatetime 进⾏去重，保留最新数据。b、双更新：更新数据时，会同时更新新旧数据源中满⾜条件的数据。c、双删除：删除数据时，会同时删除新旧数据源中满⾜条件的数据。双删的⽬的是，为了避免只 删除了⼀个库的数据，⽽由于双查导致另⼀个库的数据被查询出来误⽤。d、单插⼊：插⼊数据时，只会插⼊新分⽚算法所落⼊的数据源中。这样做的好处是，在不影响业 务的情况下，也减少了⼀部分数据迁移的⼯作量，即分库上线后新数据直接插⼊新库，不⽤做任何处理。\n\n改造源码如下：\n\n```java\npackage org.apache.shardingsphere.core.rule;\n\n/**\n * Databases and tables sharding rule.\n */\n@Getter\npublic class ShardingRule implements BaseRule {\n\n    // 定义：旧的分库分片策略\n    private final ShardingStrategy oldDatabaseShardingStrategy;\n   \n    public ShardingRule(final ShardingRuleConfiguration shardingRuleConfig, final Collection<String> dataSourceNames) {\n        Preconditions.checkArgument(null != shardingRuleConfig, \"ShardingRuleConfig cannot be null.\");\n        Preconditions.checkArgument(null != dataSourceNames && !dataSourceNames.isEmpty(), \"Data sources cannot be empty.\");\n        this.ruleConfiguration = shardingRuleConfig;\n        shardingDataSourceNames = new ShardingDataSourceNames(shardingRuleConfig, dataSourceNames);\n        tableRules = createTableRules(shardingRuleConfig);\n        broadcastTables = shardingRuleConfig.getBroadcastTables();\n        bindingTableRules = createBindingTableRules(shardingRuleConfig.getBindingTableGroups());\n        // 加载旧的分库分片策略\n        ShardingStrategyConfiguration oldDatabaseShardingStrategyConfig = shardingRuleConfig.getOldDatabaseShardingStrategyConfig();\n        if (oldDatabaseShardingStrategyConfig != null) {\n            oldDatabaseShardingStrategy = createDefaultShardingStrategy(oldDatabaseShardingStrategyConfig);\n        } else {\n            oldDatabaseShardingStrategy = null;\n        }\n        defaultDatabaseShardingStrategy = createDefaultShardingStrategy(shardingRuleConfig.getDefaultDatabaseShardingStrategyConfig());\n        defaultTableShardingStrategy = createDefaultShardingStrategy(shardingRuleConfig.getDefaultTableShardingStrategyConfig());\n        defaultShardingKeyGenerator = createDefaultKeyGenerator(shardingRuleConfig.getDefaultKeyGeneratorConfig());\n        masterSlaveRules = createMasterSlaveRules(shardingRuleConfig.getMasterSlaveRuleConfigs());\n        encryptRule = createEncryptRule(shardingRuleConfig.getEncryptRuleConfig());\n    }\n}\n```\n\n\n\n```java\npackage org.apache.shardingsphere.sharding.route.engine;\n\n/**\n * Sharding route decorator.\n */\npublic final class ShardingRouteDecorator implements RouteDecorator<ShardingRule> {\n    \n    @SuppressWarnings(\"unchecked\")\n    @Override\n    public RouteContext decorate(final RouteContext routeContext, final ShardingSphereMetaData metaData, final ShardingRule shardingRule, final ConfigurationProperties properties) {\n        SQLStatementContext sqlStatementContext = routeContext.getSqlStatementContext();\n        // SQL参数集合\n        List<Object> parameters = routeContext.getParameters();\n        ShardingStatementValidatorFactory.newInstance(\n                sqlStatementContext.getSqlStatement()).ifPresent(validator -> validator.validate(shardingRule, sqlStatementContext.getSqlStatement(), parameters));\n        // 获取分片条件\n        ShardingConditions shardingConditions = getShardingConditions(parameters, sqlStatementContext, metaData.getSchema(), shardingRule);\n        boolean needMergeShardingValues = isNeedMergeShardingValues(sqlStatementContext, shardingRule);\n        if (sqlStatementContext.getSqlStatement() instanceof DMLStatement && needMergeShardingValues) {\n            checkSubqueryShardingValues(sqlStatementContext, shardingRule, shardingConditions);\n            mergeShardingConditions(shardingConditions);\n        }\n        ShardingRouteEngine shardingRouteEngine = ShardingRouteEngineFactory.newInstance(shardingRule, metaData, sqlStatementContext, shardingConditions, properties);\n        RouteResult routeResult = null;\n        // ShardingStandardRoutingEngine特殊处理\n        if(shardingRouteEngine instanceof ShardingStandardRoutingEngine) {\n            SQLStatement sqlStatement = sqlStatementContext.getSqlStatement();\n            // 查询/更新/删除操作，开启双操作模式\n            boolean isCompatibleMode = (sqlStatement instanceof SelectStatement) || (sqlStatement instanceof UpdateStatement) || (sqlStatement instanceof DeleteStatement);\n            routeResult = shardingRouteEngine.route(shardingRule, isCompatibleMode);\n        } else {\n            routeResult = shardingRouteEngine.route(shardingRule);\n        }\n        if (needMergeShardingValues) {\n            Preconditions.checkState(1 == routeResult.getRouteUnits().size(), \"Must have one sharding with subquery.\");\n        }\n        return new RouteContext(sqlStatementContext, parameters, routeResult);\n    }\n}\n```\n\n\n\n```java\npackage org.apache.shardingsphere.sharding.route.engine.type.standard;\n\n/**\n * Sharding standard routing engine.\n */\npublic final class ShardingStandardRoutingEngine extends BaseShardingRoutingEngine {\n\n    /**\n     * 寻找sharding路由结点信息\n     * @param shardingRule\n     * @param tableRule\n     * @param databaseShardingValues\n     * @param tableShardingValues\n     * @param isCompatibleMode 兼容模式（即同时操作新旧分片算法定位到的数据库，查询/更新/删除自动开启兼容模式）\n     * @return\n     */\n    private Collection<DataNode> route0(final ShardingRule shardingRule, final TableRule tableRule, final List<RouteValue> databaseShardingValues, final List<RouteValue> tableShardingValues,\n                                        final boolean isCompatibleMode) {\n        Collection<DataNode> result = new LinkedList<>();\n        // 双查 & 双更 & 双删核心逻辑\n        // 1. 加载旧数据节点路由信息\n        ShardingStrategy oldDatabaseShardingStrategy = shardingRule.getOldDatabaseShardingStrategy();\n        Collection<String> oldDataSources = null;\n        if (oldDatabaseShardingStrategy != null && isCompatibleMode) {\n            oldDataSources = routeDataSources(shardingRule, tableRule, databaseShardingValues, true);\n            for (String each : oldDataSources) {\n                result.addAll(routeTables(shardingRule, tableRule, each, tableShardingValues, true));\n            }\n        }\n        // 2. 加载新数据节点路由信息\n        Collection<String> routedDataSources = routeDataSources(shardingRule, tableRule, databaseShardingValues, false);\n        for (String each : routedDataSources) {\n            // 新数据节点未被加载时，加载新数据节点\n            if(oldDataSources == null || !oldDataSources.contains(each)) {\n                result.addAll(routeTables(shardingRule, tableRule, each, tableShardingValues, false));\n            }\n        }\n        return result;\n    }\n    \n    private Collection<String> routeDataSources(final ShardingRule shardingRule, final TableRule tableRule, final List<RouteValue> databaseShardingValues, final boolean isOld) {\n        if (databaseShardingValues.isEmpty()) {\n            return tableRule.getActualDatasourceNames();\n        }\n        // 根据isOld参数，区分加载新/旧分片策略\n        Collection<String> result = new LinkedHashSet<>();\n        if (isOld) {\n            result.addAll(shardingRule.getOldDatabaseShardingStrategy().doSharding(tableRule.getActualDatasourceNames(), databaseShardingValues, this.properties));\n        } else {\n            result.addAll(shardingRule.getDatabaseShardingStrategy(tableRule).doSharding(tableRule.getActualDatasourceNames(), databaseShardingValues, this.properties));\n        }\n        Preconditions.checkState(!result.isEmpty(), \"no database route info\");\n        Preconditions.checkState(tableRule.getActualDatasourceNames().containsAll(result), \n                \"Some routed data sources do not belong to configured data sources. routed data sources: `%s`, configured data sources: `%s`\", result, tableRule.getActualDatasourceNames());\n        return result;\n    }\n\n}\n```\n\n **4.2数据分片**基于业务进⾏分库的分⽚键选择，最好能均匀的将数据拆分到不同数据库实例。本⽅案假设选择⽤⼾ID作为分库的分⽚键，然后⾃定义新的分库分⽚算法，**算法命名为 DatasourceShardingAlgorithmV2**，算法内容如下：\n\n```java\nimport org.apache.shardingsphere.api.sharding.standard.PreciseShardingAlgorithm;\nimport org.apache.shardingsphere.api.sharding.standard.PreciseShardingValue;\nimport java.nio.ByteBuffer;\nimport java.nio.ByteOrder;\nimport java.util.*;\n\n/**\n * 新的分库数据分片算法\n *\n * @author jingmin.yang\n * @date 2022/4/18 19:23\n */\npublic class DatasourceShardingAlgorithmV2 implements PreciseShardingAlgorithm<String> {\n    // 排序存储结构：SortedMap<虚拟节点，物理节点>\n    private static final SortedMap<Integer, String> virtualToRealMap = new TreeMap<Integer, String>();\n\n    /**\n     * 基于所有数据源初始化订单分库虚拟节点\n     * @return\n     */\n    static {\n        List<String> datasources = new ArrayList<String>();\n        datasources.add(\"datasource-name\");\n        datasources.add(\"datasource-name-1\");\n        datasources.add(\"datasource-name-2\");\n        datasources.add(\"datasource-name-3\");\n        // 初始化分库虚拟节点数据\n        init(virtualToRealMap, datasources);\n    }\n\n    @Override\n    public String doSharding(Collection<String> availableTargetNames, PreciseShardingValue<String> preciseShardingValue) {\n        // shardingKey即为分片键值\n        String shardingKey = preciseShardingValue.getValue();\n        return getShardingDatasource(virtualToRealMap, shardingKey);\n    }\n\n    /**\n     * 获取分库物理节点\n     * @param virtualToRealMap 分库虚拟节点\n     * @param shardingKey 分库的分片键标识\n     * @author: jingmin.yang\n     * @create: 2021/9/10 10:41\n     * @return\n     */\n    static String getShardingDatasource(SortedMap<Integer, String> virtualToRealMap, String shardingKey) {\n        int hashValue = hash(shardingKey);\n        // 顺时针找最近的虚拟节点\n        SortedMap<Integer, String> subVirtualToRealMap = virtualToRealMap.tailMap(hashValue);\n        if (subVirtualToRealMap.isEmpty()) {\n            return virtualToRealMap.get(virtualToRealMap.firstKey());\n        }\n        return subVirtualToRealMap.get(subVirtualToRealMap.firstKey());\n    }\n\n    /**\n     * 初始化hash环\n     *\n     * @param: virtualToRealMap\n     * @param: realNodes\n     * @author: jingmin.yang\n     * @create: 2021/9/14 16:29\n     */\n    static void init(SortedMap<Integer, String> virtualToRealMap, List<String> realNodes) {\n        for(String node: realNodes) {\n            virtualToRealMap.put(hash(node), node);\n\n            // 虚拟出指定数量的虚拟节点\n            int count = 0;\n            int i = 0;\n            int virtualNum = realNodes.size();\n            while (count < virtualNum) {\n                i++;\n                String virtualNode = node + \"#\" + i;\n                // 计算hash值\n                int hashValue = hash(virtualNode);\n                if (!virtualToRealMap.containsKey(hashValue)) {\n                    // 虚拟节点放到环上去 数据结构：红黑树\n                    virtualToRealMap.put(hashValue, node);\n                    count++;\n                }\n            }\n        }\n    }\n\n    /**\n     *\n     * MurmurHash是一种非加密型哈希函数，适用于一般的哈希检索操作,与其它流行的哈希函数相比，对于规律\n     * 性较强的key，MurmurHash的随机分布（离散性）特征表现更好，而且具有低碰撞率。Redis对节点进行\n     * shard时采用的是这种算法。\n     *\n     * @param: key 分片键\n     * @author: jingmin.yang\n     * @create: 2021/9/14 19:17\n     */\n    static int hash(String key) {\n        ByteBuffer buf = ByteBuffer.wrap(key.getBytes());\n        int seed = 305441741;\n        ByteOrder byteOrder = buf.order();\n        buf.order(ByteOrder.LITTLE_ENDIAN);\n        int m = 1540483477;\n        byte r = 24;\n\n        int h;\n        int k;\n        for (h = seed ^ buf.remaining(); buf.remaining() >= 4; h ^= k) {\n            k = buf.getInt();\n            k *= m;\n            k ^= k >>> r;\n            k *= m;\n            h *= m;\n        }\n\n        if (buf.remaining() > 0) {\n            ByteBuffer finish = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN);\n            finish.put(buf).rewind();\n            h ^= finish.getInt();\n            h *= m;\n        }\n\n        h ^= h >>> 13;\n        h *= m;\n        h ^= h >>> 15;\n        buf.order(byteOrder);\n        if (h < 0) {\n            h = Math.abs(h);\n        }\n        return h;\n    }\n}\n```\n\n测试： \n\n```java\npublic static void main(String[] args) {         \n    System.out.println(DatasourceShardingAlgorithmV2.getShardingDatasource(virtualToRealMap, \"U0000000001\"));\n}\n```\n\n结果：\n\n```Plaintext\ndatasource-name-1\n```\n\n **4.3规则配置**Sharding-JDBC 可以通过 Java, YAML, Spring 命名空间 和 Spring Boot Starter 这 4种⽅式进⾏配置，开发者可根据场景选择适合的配置⽅式。当前⽅案使⽤ YAML ⽅式进⾏规则配置，application.ym 内容如下（**红⾊部分为⾃⾏扩展内容**）： \n\n```yaml\nspring: \n shardingsphere:\n  # sharding数据源配置\n  datasource:\n    # 对应下方自定义的数据源名称\n    names: datasource-name,datasource-name-1,datasource-name-2,datasource-name-3\n    # 数据源基础配置\n    datasource-name:\n     # 数据源驱动类型\n      type: com.alibaba.druid.pool.DruidDataSource\n      # 数据库驱动类\n      driver-class-name: com.mysql.jdbc.Driver\n      # 数据库连接地址\n      url: jdbc:mysql://127.0.0.1:3306/datasource_name_0\n     # 数据库用户名\n      username: username\n     # 数据库用户密码\n      password: password\n    # 自定义第二个数据源（新数据源1）\n    datasource-name-1:\n      type: com.alibaba.druid.pool.DruidDataSource\n      driver-class-name: com.mysql.jdbc.Driver\n      url: jdbc:mysql://127.0.0.1:3306/datasource_name_1\n      username: username\n      password: password\n    # 自定义第三个数据源（新数据源2）\n    datasource-name-2:\n      type: com.alibaba.druid.pool.DruidDataSource\n      driver-class-name: com.mysql.jdbc.Driver\n      url: jdbc:mysql://127.0.0.1:3306/datasource_name_2\n      username: username\n      password: password\n    # 自定义第四个数据源（新数据源3）\n    datasource-name-3:\n      type: com.alibaba.druid.pool.DruidDataSource\n      driver-class-name: com.mysql.jdbc.Driver\n      url: jdbc:mysql://127.0.0.1:3306/datasource_name_3\n      username: username\n      password: password\n  # 采用sharding-jdbc进行分库分表设计\n  sharding:\n    # 默认数据源\n    default-data-source-name: datasource-name\n    # oldDatabaseStrategy是自行扩展配置（可选），用于双操作（双查询&双更新&双删除）中获取原有数据源，pom需要对应引入4.1.1.001自行扩展版本\n    oldDatabaseStrategy:\n      standard:\n        # 分库分片键\n        shardingColumn: sharding_key\n        # 原有的数据源分片算法，本算法返回的是固定的原有数据源datasource-name\n        preciseAlgorithmClassName: com.yang.sharding.DatasourceShardingAlgorithm\n   # 订单分库新的分片算法，根据sharding_key进行分库\n   default-database-strategy:\n      standard:\n        # 分库分片键\n        shardingColumn: sharding_key\n        # 新的分库分片算法\n        preciseAlgorithmClassName: com.yang.sharding.DatasourceShardingAlgorithmV2\n    # 分库分表配置，只有分库&分表的表需要在此配置，不分库不分表不需要配置，数据会存入默认数据源\n    tables:\n      # table_0使用分库+分表\n      table_0:\n        actual-data-nodes: datasource-name.table_0_$->{0..63},datasource-name-$->{1..3}.table_0_$->{0..63}\n        logic-table: table_0\n        # 数据库表策略配置\n        table-strategy:\n          standard:\n            # 表分片键\n            shardingColumn: sharding_key\n            # 表分片算法\n            preciseAlgorithmClassName: com.yang.TableShardingAlgorithm\n      # table_1使用分库，不使用分表\n      table_1:\n        actual-data-nodes: datasource-name.table_1, datasource-name-$->{1..3}.table_1\n        logic-table: table_1\n```\n\n**4.4代码改造准备**由于 sharding 是对分⽚键使⽤分⽚算法，获取到对应的数据源进⾏操作，⽽分⽚键是从请求 sql 中提取⽽来，所以需要在代码层⾯改造⽀持分库，所有 CRUD 操作必须要带上分⽚键参数。\n\n```sql\n<!-- 这里假设数据库是MySQL，分片键是user_id字段 -->\n<!-- 新增操作 -->\nINSERT INTO order_info('id', 'user_id', 'update_time') VALUES('O0000000001', 'U0000000001', NOW());\n<!-- 查询操作 -->\nSELECT * FROM order_info WHERE user_id='U0000000001';\n<!-- 更新操作 -->\nUPDATE order_info SET update_time=NOW() WHERE user_id='U0000000001';\n<!-- 删除操作 -->\nDELETE FROM order_info WHERE user_id='U0000000001';\n```\n\n \n\n# **总结**\n\n**⾄此，基于 Sharding-JDBC 完整的订单分库改造完成，包含选择分库分⽚键、定义分库数据分⽚算法，添加数据源&表配置，代码改造等。**本⽅案已有实际应⽤案例，如有需要的⼩伙伴可作为参考，有问题也可直接联系我，⼀起讨论学习。","date":"2022-04-24","author":"杨敬敏 Ginee后端工程师","excerpt":"本⽅案基于Apache开源项⽬ ShardingSphere 采 Sharding-JDBC 进⾏订单⽔平分库设计（数据分⽚）缓解存储压⼒。","metaDescription":null,"createdAt":"2023-03-29T07:36:03.794Z","updatedAt":"2023-03-29T07:36:04.607Z","publishedAt":"2023-03-29T07:36:04.603Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":38,"attributes":{"feature":true,"title":"金鹰系统数据库分库分表策略","content":"为了全面监测区域内企业的金融风险，金鹰系统汇集了工商、舆情、投诉举报等多维度的大量数据。随着时间的推移，积攒的数据量越来越大，对系统的数据处理效率提出了更高的要求。目前金鹰系统单表最大数据量已达到亿级别，传统的数据库性能优化手段已无法满足系统的性能需求。为了减少数据库的负担，提升数据库响应速度，缩短查询时间，防控中心技术团队对数据进行分库分表。\n\n**什么是分库分表？**\n\n分库指从单个数据库拆分成多个数据库的过程，将数据散落在多个数据库中。而分表指从单张表拆分成多张表的过程，将数据散落在多张表内。\n\n**分库分表能解决什么问题？**\n\n如果数据库的查询压力过大，就需要考虑拆库，通过分库来分担单个数据库的连接压力。比如，数据库每秒需处理3500笔请求，假设单库每秒能处理的请求数最大为1000笔，那么就可以考虑拆分成4个库，来分散查询连接压力。\n\n如果单表数据量过大，当数据量超过一定量级后，无论是对于数据查询还是数据更新，在经过索引优化等纯数据库层面的传统优化手段之后，还是可能存在性能问题。这时候就需要去换个思路来解决问题。比如：从数据生产源头、数据处理源头来解决问题，既然数据量很大，那我们就来个分而治之，化整为零。可以通过分表把数据按照一定的规则拆分成多张表，来解决单表环境下无法解决的存取性能问题。\n\n►►►\n\n**金鹰系统数据库分库分表策略**\n\n金鹰系统目前采用ShardingSphere（一款分库分表框架）作为数据库中间件。通过对企业名称、法人等多个关键字进行组合散列运算得到企业ID。以企业ID作为分片键，ShardingSphere将根据企业ID与配置的路由信息决定该企业的数据落于哪个数据库分片、哪张表上，合理的算法使数据均匀分布在各数据库分片上，不同分片保存的数据量大致相同，从而确保数据处理效果能得到最大的优化。 \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MGM5YWQ1NmU0MzBlNzFiYzhmMjYyN2MyNjk5MTE5ZThfQzlkaERYQzZWTWVWRFBkNkRYTm1WcG13eWtlZXVDZzVfVG9rZW46Ym94Y25QSGdXUEwwaTQwenB6eWRoajFIU2xlXzE2ODAwNzM5MzQ6MTY4MDA3NzUzNF9WNA)","date":"2022-04-21","author":"广州金融科技","excerpt":"金鹰系统通过对监管企业的海量信息数据采用数据分库分表方式治理，提高了数据查询、数据处理效率","metaDescription":null,"createdAt":"2023-03-29T07:12:54.670Z","updatedAt":"2023-03-29T07:12:56.574Z","publishedAt":"2023-03-29T07:12:56.571Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":42,"attributes":{"feature":true,"title":"智能电站数据中台建设与应用","content":"## **引言**\n\n \n\n近几年随着计算机技术、通信技术、人工智能技术的发展，电站正逐步向网络化、数字化、智能化迈进。智能电站是以电站生产和管理活动中各种信息的数字化为前提，计算机、网络、通信技术为基础，通过采用数据分析和处理、智能控制、自动决策等技术，使得在各种环境条件下都能够提供经济、环保、安全、稳定的电能并与智能电网高度融合的发电站([1-3])。数据信息资源正逐步成为电站的关键要素资源。由于电站信息系统大多是分阶段、分功能实施的，在电站智能化建设过程中，特别是在数据资源使用过程中暴露出来了许多问题。（1）数据孤岛现象严重。各个业务系统数据各自存储、各自定义，数据无法有效连接，数据分散、指标重复建设、数据实用价值低的现象普遍存在。（2）数据缺乏全生命周期管理。电站设计、基建阶段产生的数据不能传递到生产、运维阶段，影响了后期机组及设备的性能诊断和运行优化分析。（3）数据能力未能积累和沉淀。数据服务化程度低，数据共享和交互上缺乏整体协同性，无法形成有效的电站数据资产框架。\n\n如何唤醒电站沉睡的数据资产、打通数据信息壁垒、消除数据烟囱，把数据融合利用起来，是电站智能化转型过程中的难题。而以构建数据资产体系、释放数据资产价值为核心的数据中台正是这一问题的完美解决方案，对提升电站精益化运行管理水平具有十分重要的意义。\n\n \n\n### **1  数据中台架构设计**\n\n \n\n**1.1  电站数据源**\n\n来自电站不同设备和系统的数据主要是生产数据和管理数据，数据类型包含结构化、半结构化和非结构化3类数据。（1）结构化数据包括时序数据和非时序数据。时序数据一般指控制系统现场传感器数据采集过来的数据，比如火电厂的锅炉设备温度、压力等传感器信号，经过规整后存入工业实时数据库中；非时序数据包括各类关键业务系统数据，如智能检修、智能燃料、智能运行、智能经营等系统数据，这些数据基于关系型数据库存储，表现为二维形式。（2）半结构化数据包括各类系统的配置文件、产生的日志文件等，这类数据一般以XML、JSON或自定义文本文件格式存储。（3）非结构化数据包括电站档案系统、门禁系统、智能摄像头、移动机器人产生的各类技术文档、图像文件、音频文件、视频文件等。电站现有平台的管理数据与生产数据没有有效融合，实时数据库与关系数据库的互联互通也存在一定问题，未能进行数据有效组织管理和数据赋能。\n\n**1.2  数据中台架构**\n\n智能电站数据中台通过构建面向电站应用的数据共享和数据分析智能平台，克服了电站现有系统的技术缺陷。数据中台以数据资产高效管理为基础，沉淀数据通用能力，通过构建规范的、安全的、智能的、全域化的数据处理架构，提供数据采集、数据存储、数据萃取、数据治理、数据服务、数据可视化等全链路一体化的服务([4-9])，其技术架构如图1所示。\n\n \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ZGEyOTI5OGNmOGY2OGIwZDJmZjM1MjE0MTU3NGE2ZDFfUEgxamM0bWJEWFY4UmllVWNhUktLWVc5SW0xeHVoR1BfVG9rZW46Ym94Y25HYkllUWh2WERZWGtwY25kMWwzNUJmXzE2ODAxNTc3MDQ6MTY4MDE2MTMwNF9WNA)\n\n<center>图1  智能电站数据中台技术架构</center>\n\n<center>Fig.1  Technology architecture of intelligent power station data middle platform</center>\n\n数据采集是数据中台接入数据的入口，把各种异构数据源的数据采集到数据中台进行集中存储；数据萃取与分析将低价值的原始业务数据转换为高价值的数据资产，通过数据资产框架进行治理；基于数据服务体系将数据以服务化方式交付给业务系统，通过数据可视化框架进行数据展示，激活整个数据中台。\n\n**1.3  数据中台价值**\n\n数据中台的价值体现在“存、通、用、统、快”，如图2所示。通过构建智能电站数据中台，打通数据隔阂，统一数据标准，实现一切业务数据化、一切数据业务化、快速响应用户需求。（1）一切业务数据化。通过业务信息系统的建设，将系统中沉淀的业务数据汇聚到数据中台进行存储，实现一切业务数据化。（2）一切数据业务化。数据中台基于业务系统产生的数据进行体系化加工，将结果以服务化方式提供给业务使用，进而反哺业务，实现一切数据业务化。 \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=Y2Y0OGY1YmU1MzhkNzljNTAxNDg2MTM1ZWQxODQ1YmJfRVZkUnNSbXNjQWpXQmN2aFpZU0NRaTJLR1ZpOE9HcFRfVG9rZW46Ym94Y25QR2tWS3JRZzExcVZkaG5BVkpHdWljXzE2ODAxNTc3MDQ6MTY4MDE2MTMwNF9WNA)\n\n<center>图2  数据中台价值</center>\n\n<center>Fig.2  Value of data middle platform</center>\n\n \n\n### **2  数据中台关键技术**\n\n \n\n**2.1  全域数据采集**\n\n全域数据采集是智能电站数据中台建设的第一步，也是非常重要的一步，为数据中台全面汇集异构数据。数据采集平台可以针对不同的数据源，建立不同的数据采集任务，具备数据采集、转换、传输、复制、迁移、同步等能力([10-11])。（1）电站每台机组控制系统大约有10万个传感器测点，时间戳精度为秒级。转子设备振动数据时间戳精度可以达到毫秒级。采用自主研发的实时数据采集传输程序，基于例外发送、数据缓存、断点续传和历史回补机制，实现电站实时数据的快速高效传输，目前支持OPC、Modbus、101~104规约、CDT规约、RTU规约及其他私有TCP/UDP通信协议。（2）电站业务关系数据库众多，其采集与同步可使用Sqoop和Kettle开源程序。抽取模式可采用全量抽取和增量抽取模式，增量数据抽取一般基于时间戳、触发器、更新标识等机制。（3）电站日志系统数据采集可采用Flume。（4）针对电站外部互联网数据，使用网页爬虫方式自动抓取目标数据。\n\n**2.2  数据湖底座建设**\n\n数据湖底座是智能电站全域数据的存储中心，针对不同数据源和不同数据应用分析需求提供灵活、可扩展的混合型存储架构，以分布式存储系统为核心，能够实现任意规模结构化、半结构化和非结构化数据存储，以提高生产力、主动维护设备，为电站的数据融合提供支撑，为智能运行、智能检修等生产决策提供依据。（1）电站机组控制系统的传感器测点数据采集频率和时间戳精度均为秒级，考虑到存储性能和后期查询、统计、分析性能，采用自主研发的睿腾（Rython）工业实时数据库进行存储，数据库系统架构如图3所示。睿腾实时数据库基于高并发网络通信技术、IO资源管控技术、多级高速数据缓存算法、无损压缩和有损压缩相结合的机制，实现海量测点数据的高效传输、存储、管理和访问。 \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=YTVlYjFlN2MxZTE2NWRhZjU1YWU3NjY2ODgyMDY0MjlfTHkxRFhNeGRhbTAwYWQ1SEF0MTBsSjV2SzJnZnhtNWtfVG9rZW46Ym94Y25uVm40WElHdHRiUHJmbFo3bU9kanVnXzE2ODAxNTc3MDQ6MTY4MDE2MTMwNF9WNA)\n\n<center>图3  睿腾实时数据库系统架构</center>\n\n<center>Fig.3  System architecture of Rython real-time database</center>\n\n \n\n睿腾实时数据库可管控的测点规模可达千万级别。测点表基于两级索引构建，其索引结构如图4所示。在此索引结构中，如果已知一个测点的记录号RecNo，就可通过以下步骤在测点表文件中进行索引定位并进行相应的操作：①计算RecNo/10240可得到该测点所属的一级索引表单元位置，并从一级索引表中读取到二级索引块的起始文件块号；②计算RecNo % 10240可得到该测点所属的二级索引块的单元位置，并从二级索引块中读取到测点属性的文件块号；③根据索引得到的测点属性文件块号，可对该测点属性进行创建和编辑操作；如果需要删除一个标签，则只需要将该测点的二级索引块单元中的标签属性文件块号的高位（第31Bits）置为1即可，不需要更新测点属性。\n\n \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ZWI2NzQ5ZjY2ZGFiYjBlYTUzOTZkYjA5NzgwYzZjODVfbVNlZEI3V2tsZ2VVelROYThBeU9NNURZVldlalhweWdfVG9rZW46Ym94Y25aVTE4eVdCZGthSkV0aTEyYTVhN0tjXzE2ODAxNTc3MDQ6MTY4MDE2MTMwNF9WNA)\n\n<center>图4  睿腾实时数据库测点表索引结构</center>\n\n<center>Fig.4  Tag’s index structure of Rython real-time database</center>\n\n（2）分布式大数据平台基于自主研发的大数据产品，以分布式文件系统为核心，由管理节点和一定数量的工作节点组成大数据存储集群，支撑大数据分析、萃取、挖掘，保存加工处理后的结构化数据和图片、文档等非结构化数据([12])。（3）电站关键业务数据采用分布式关系数据库进行存储。关系数据库平台基于改进的MySql+MHA的部署方案，提供高可用、高数据吞吐能力、扩展性强的关系型数据库引擎。对于分布式数据访问层，采用开源shardingjdbc技术路线。\n\n**2.3  数据萃取与分析**\n\n基于数据开发和算法开发平台，数据萃取层通过电站业务主题域和数据仓库建设，构建完整的智能电站测点数据体系、指标数据体系、设备数据体系、成本数据体系、人员数据体系和文档数据体系。数据开发是将价值不大的原始数据变成对业务有价值的数据资产。数据开发平台产品提供批量计算、流式计算和查询分析能力([13])。（1）批量计算基于MapReduce、Spark等计算框架进行处理，主要用于人机交互较少的批量数据高延时处理，如离线数据仓库的加工、大规模数据清洗与挖掘等，其数据吞吐量大、延时高。（2）流式计算基于Storm、Spark Streaming等计算框架进行处理，数据源为源源不断的流式数据，适用于对数据的加工处理和应用有较高时效性的场景，如设备异常监控告警。（3）查询分析使用ElasticSearch、Kylin、ClickHouse等技术进行处理，适用于历史数据的在线查询、条件过滤和维度分析。算法开发作为一站式的企业机器学习开发平台，包括数据集管理（如数据接入与标注等）、核心算法组件（如回归、聚类、神经网络等）编写、主流算法框架（如TensorFlow、PyTorch、Mahout等）集成、可视化建模，旨在为企业快速提供核心算法服务([14])。基于自主研发的分布式图形化建模计算平台，通过简单的拖拽操作，就可组态出复杂的算法逻辑，为数据中台提供强大的数据建模和计算能力。图形化建模计算平台主要包含以下功能。（1）模型设计器：提供可视化的机理建模、数学建模工具。（2）工程组态：组态工程的编辑、调试、上线、维护管理。（3）算法因子：提供基础通用算法、热力学专用算法、大数据算法等不同种类算法因子。同时可以自定义算法，具备集成第三方算法的能力。（4）计算引擎：基于时间、条件、公式等基础引擎，采用流式计算、离线计算、内存计算等计算模式，进行分布式在线和离线计算。\n\n**2.4  数据资产治理**\n\n数据治理框架，为智能电站提供统一数据标准制定、主数据管理、元数据管理、数据质量管理等能力，通过数据地图和数据血缘实现电站数据360°全链路追踪。框架提供数据全生命周期的管控机制，当发生数据质量问题时，可通过可视化图形方式追溯指标的血缘及逻辑关系、关联指标的数据和权重，进而定位问题([15])。（1）电站资产模型通过设备树的方式进行组织定义，每个设备都用元素表示，元素的关联数据将作为元素的属性进行存储，元素属性数据包括常量、实时数据库测点值、关系数据库引用值、指标公式计算值、图片、文档等。（2）数据治理计算框架基于分布式消息引擎RocketMQ和分布式缓存Redis集群，采用事件触发和周期触发机制，实现设备树海量元素属性数据的层次计算和分析计算。**2.5  数据服务化交付**智能电站数据中台将数据进行服务化交付，企业内部业务系统和第三方应用不必关心底层数据库结构，就可以获得目标数据。通过这样的架构设计，对下屏蔽了异构数据的差异性，对上提供了统一的数据服务接口和标准化数据。当新的业务需求出现时，可以快速进行开发响应。基于自主研发的数据服务网关，可以为业务应用提供以下3种形式的数据服务。（1）OPC UA数据服务。在应用端对数据访问效率要求极高时，如实时性很强的测点数据，推荐使用OPC UA服务。（2）Restful WebAPI数据服务。基于微服务架构和SpringCloud技术体系，提供基于HTTP/HTTP2的服务接口。（3）文件数据服务。基于FastDFS分布式文件系统，为业务应用提供在线文档服务。\n\n**2.6  数据可视化框架**\n\n数据可视化框架基于Vuejs前端开源javascript库和CSS响应式布局模式构建，框架设计器如图5所示。框架组件设计采用MVVM（model view viewmodel）模式进行定义，框架内部基于HTML5、Canvas和WebGL图形绘制技术，实现了大量的2D、3D、VR等图形组件。框架组件间消息通讯采用Weak Event机制实现，这样不仅降低了组件间的业务耦合度，也消除了内存泄漏的风险。开发人员或业务人员使用数据可视化框架能够快速进行生产运营决策驾驶舱的数据准备、指标集建立、业务模型配置、管理视图及可视化图表构建，实现对电站海量数据的汇总统计、多维分析、报表呈现、图形展示，以交互、生动的形式直观、全面、快速地了解电厂生产经营情况并进行辅助决策，进而提升工作效率。 \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=Mjg5OGFlZWUyNTljOTYxMWJjZGRjMWQ1NDhmNGNmNDlfTUhaazU0VTFvT1d6WWt5RjcxNDJVOUtaUGk1R1VTNHFfVG9rZW46Ym94Y25XaUJJUGJNRWdYa1pDOGZJVWkyODliXzE2ODAxNTc3MDQ6MTY4MDE2MTMwNF9WNA)\n\n<center>图5  数据可视化框架设计器</center>\n\n<center>Fig.5  Designer of data visualization framework</center>\n\n \n\n### **3  数据中台应用**\n\n \n\n建设智能电站数据中台的最终目的是高效组织利用数据资源，提升电站精益化运行管理水平。以智能电站建设过程中的3个典型应用场景为例，阐述企业如何应用数据中台，提高数据治理、分析计算和展示能力。\n\n**3.1  设备运行监视**\n\n电站在运行过程中，需要对锅炉、汽轮机、发电机、磨煤机、引风机等关键主辅机设备进行实时监控。设备数据种类复杂、来源众多，以磨煤机为例，主要监视的信息如下。（1）一次测点数据。这些数据来自现场控制系统传感器，如磨煤机电流、功率、进口风量、出口风粉温度等，采集频率为秒级，为时间序列型数据。（2）指标数据。这些数据由不同业务系统产生，如发电量、利用小时、非计划停运次数等指标来自可靠性系统，而发电量、负荷率、耗煤量等指标来自在线经营系统。指标数据均通过一次测点数据不同的运算逻辑统计计算得到。（3）设备基础信息。这些数据主要来自设备管理系统，主要包括设备名称、编码、设备KKS编码、所属专业、设备分类、型号规格、生产厂家、出厂编号、安装地点、安装现场图片等信息。（4）设备检修和缺陷记录。这些信息主要来自ERP系统（如SAP），包括检修发生时间、故障描述、参与人员、耗费人工、使用物料、消缺状态等。在传统信息系统建设方式下，电站的各种系统大多是独立建设的，存在指标重复计算、统计口径不一致、数据信息无法互联互通等问题，也无法很好地应对快速变化的前端业务。通过建设智能电站数据中台，可以消除数据壁垒、统一数据标准，实现异构数据融合和设备运行监视的数据可视化快速配置展示。\n\n**3.2  脱硫优化系统**\n\n当前环保压力日益增大，在保证烟气二氧化硫排放达标的前提下，实现脱硫系统的安全、经济、优化运行日益重要。脱硫优化系统基于智能电站数据中台，采用数据驱动模式和大据挖掘技术，进行机组工况的规律发现、特征抽取、自动识别和智能寻优，给运行人员提供及时精细的操作指导，实现企业效益的最大化([16-18])。（1）确定优化关联指标。对脱硫系统的历史运行原始数据进行多维相关性分析，确定与寻优目标“浆液量”强相关的指标：机组负荷、入口二氧化硫浓度、入口烟气流量、入口烟气温度、入口烟气湿度、吸收塔浆液pH值和出口二氧化硫浓度。（2）采集机组稳定运行工况。一般机组负荷波动不超过15%、至少持续运行1 h认为是一个稳定工况。基于机组运行的海量数据，在数据中台萃取层配置提取规则，进行脱硫系统优化关联指标数据的采集和统计，存储到中台数据湖底座。（3）工况标签化分类。采用中台提供的聚类算法对不同负荷工况进行分类，并对聚类类心进行合理选择。（4）以聚类获得的类心作为训练样本，采用中台提供的有监督机器学习算法建模，如图6所示，建立脱硫系统优化模型([19-21])。 \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ZTI2NzRmNjIxYjViYWJhOWJhYWQ2ZDRjZGY4Yzk2ZTdfcUp4S1VMV0psektTeVJHMjBwZTNjMTdwcG9Sd3EwVzBfVG9rZW46Ym94Y25VWHp0eWpyVjRFNkM4Y2dyZ3lwTEFlXzE2ODAxNTc3MDQ6MTY4MDE2MTMwNF9WNA)\n\n<center>图6  脱硫系统优化机器学习模型</center>\n\n<center>Fig.6  Machine learning model of desulfurization system optimization</center>\n\n（5）根据实时采集的机组负荷、入口二氧化硫浓度、入口烟气流量、入口烟气温度、入口烟气湿度、吸收塔浆液pH值和出口二氧化硫浓度数据，与稳定工况进行匹配，获得期望的“浆液量”优化目标值。\n\n**3.3  大屏幕和移动应用发布**\n\n大数据时代，电站业务数据场景分析和终端展示呈多样化发展。企业通过大屏幕发布系统，可以建立良好的形象宣传窗口；通过终端移动应用，可以随时随地查询生产实时数据、浏览生产过程图形、监控主要设备状态，对关键运营指标进行分析、钻取和统计分析。基于中台数据治理和可视化框架，采用海量酷炫图表组件和所见即所得数据源配置，用户可以快速进行大屏幕和移动应用功能设计。图7展示了智能电站“发电量分析”移动应用业务场景。 \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MmZiY2E1OGUxYjNjODliNjNmNWRiMTNjOTI4OTE5YTRfczBFWnhVRVVweGRBcXJTVzIwcWg2TzhYRXhzTVdjYkpfVG9rZW46Ym94Y25yZzV1UU5kZktJRmlHRWh2czdTY2xsXzE2ODAxNTc3MDQ6MTY4MDE2MTMwNF9WNA)\n\n<center>图7  智能电站移动应用</center>\n\n<center>Fig.7  Mobile app of intelligent power station</center>\n\n \n\n### **4  结语**\n\n \n\n（1）针对电站现有系统存在的数据孤岛、数据未能有效利用问题，分析了电站发电过程中产生的海量数据的数据源类型，设计了智能电站数据中台架构，消除了数据烟囱，统一了数据标准，实现了多源异构数据融合。（2）以数据资产高效治理为基础，通过全域数据采集、数据湖底座建设、数据萃取与分析、数据服务化交付、数据可视化等关键技术方案设计，构建和实现了智能电站数据中台。（3）介绍了智能电站数据中台的典型应用场景，阐述了如何应用数据中台提高数据治理、分析计算和展示能力，对于发电企业建设和应用数据中台具有一定的借鉴意义。","date":"2022-04-20","author":"王毅, 王智微, 何新","excerpt":"在电站智能化建设过程中，为解决现有系统的数据孤岛、数据利用率低等问题，设计了数据中台产品。基于全域数据采集、数据湖底座建设、数据萃取与分析、数据服务化交付、数据可视化关键技术，建立了智能电站数据中台，实现了多源异构数据融合、共享和服务能力，有效提升电站精益化运行管理水平。最后介绍了智能电站数据中台的典型应用场景，对发电企业建设和应用数据中台具有一定的参考价值。","metaDescription":null,"createdAt":"2023-03-30T06:40:05.006Z","updatedAt":"2023-03-30T06:41:58.790Z","publishedAt":"2023-03-30T06:41:58.787Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":43,"attributes":{"feature":true,"title":"OPPO云数据库访问服务技术揭秘","content":"## **1、背景**\n\nMySQL是OPPO使用最广泛的关系数据库，不同编程语言的微服务都是通过MySQL官方的SDK直连真实的数据库实例。这种最传统的使用方式，会给业务开发和数据库运维带来一系列影响效率和稳定性的问题。\n\n- 不合理的数据库访问，比如不恰当的连接池设置、高危SQL无拦截、无限流和熔断等治理能力\n- 无弹性伸缩能力，单机数据库性能或者容量不足时，扩容非常繁琐低效\n- 缺乏常用的功能特性，不支持读写分离、影子库表、单元化架构、数据库加密\n- 不支持跨语言，并且与应用服务强耦合，编码和升级都非常困难\n\n以上问题，我们讲通过CDAS来解决单MySQL带来的一系列问题。\n\n## **2、CDAS产品简介**\n\nMySQL在并发能力、稳定性、高可用方面久经考验，是绝大多数网联网产品首选的OLTP场景存储组件，但单机MySQL在超高并发、海量存储、OLAP能力上先天不足，因此当数据到达一定量级后一般采用分库分表的方式来水平扩展MySQL，提升系统的整体处理能力。CDAS基于分片理念设计，目的是解决MySQL单机性能瓶颈，对外提供超过并发、海量存储、支持HTAP的MySQL服务。在一组MySQL集群前搭建一套高可用的代理 + 计算集群，提供分片能力、自动化的弹性伸缩能力、读写分离、影子库、数据加密存储能力，为用户提供一体化的产品。\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ZjhkYTExOTEyN2EwNjBlYzE3YjIzZjEzZDRhMGYyMTZfM3dFcXZ0d1RCN2FjVGVxeXpYbWJ2ZTJnNGFuNGFRd1VfVG9rZW46Ym94Y25tWlhpYU85Wjcxb2RVM2NEUlF0SXNiXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\nCDAS Proxy使用Java语言开发，基于开源产品Apache ShardingSphere Proxy改造而来，并添加了许多高级特性来支持内部业务。我们的开发理念是依托开源的成熟产品来搭建服务，从社区汲取营养的同时也把发现的问题、特性回馈给社区，积极参与、和社区共同发展。 \n\n**2.1 CDAS产品特点**\n\n**（1）稳定性**\n\nCDAS进行了大量的基准测试、调优，实现了特殊的动态队列 + 线程池模型来保证Proxy的总体并发程度不会由于线程数过多导致上下文切换频繁从而性能下降，同时Proxy在长时间高负载的场景下不会产生Full GC导致业务中断，目前正式环境已有多个QPS:5000+的业务场景使用了CDAS，表现稳定。\n\n**（2）高度可扩展**\n\n我们建议业务在申请分片表时预估未来3~5年的数据总量来设定分片总数，前期可申请1~2个数据库，数据量变大后扩容。理论上可扩展性和分片总数相关，最大可扩展分片数相等的数据库实例，最大可扩展到250TB，同时Proxy集群也支持在线扩容。\n\n**（3）平台化运维**\n\n由云平台统一运维、部署，MySQL和Proxy都支持自动化流程申请和变更，自动接入元数据管理系统。\n\n**（4）兼容MySQL语法**\n\n支持绝大多数MySQL语法。\n\n- 路由至单数据分片100%兼容\n- 支持分页、去重、排序、分组、聚合、关联等常见查询场景\n\n**（5）功能丰富**\n\n- 多种分片算法\n- 自增分布式ID\n- 读写分离，可设置主从同步容忍阈值\n- 影子库\n- HTAP\n- 完善的监控信息：审计日志、慢日志、鉴权 \n\n## **3.核心设计** \n\nCDAS的核心目标是解决用户海量数据存储、访问问题，聚焦到分片场景中，主要体现在解析、路由、重写、聚合等逻辑 。\n\n**3.1 内核架构**\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=Y2E0NjkzNmEyZDdmMDI1ODhjZmIxMzc1NWEyMzMwZTFfbm00REp3aU9kbFhXRHNCVVdtUTlmcm1ETHlkVVhQcmNfVG9rZW46Ym94Y25HZmtiRWdWSW9MR0RjVUZqRjhDaWVnXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n站在高处看Proxy内核，主要分为连接接入模块、I/O多路复用模块、解析路由执行引擎3个大的部分，左侧可以看到线程模型也分为3块，实际涉及线程资源的还有其他子逻辑，如并行执行线程池等，接下来将围绕多个模型和执行流程了解内核架构的细节\n\n**3.2 线程模型**内核入口层基于Netty实现，从整体上看是一个典型的Netty使用场景，内核图从可以看到主要分为3个部分。 \n\n**（1）Boss Thread**负责Accept connection，也就是接受、建立连接，客户端一般都会使用连接池技术，因此建立连接的请求不会太多，单线程即可处理 \n\n**（2）I/O Threads**即Netty EventLoopGroup ，负责编解码、发起auth认证，基于Epoll事件驱动 I/O多路复用，一个线程可处理多个连接，共CPU核心数 * 2个线程。 \n\n**（3）Worker Threads**负责执行、回写数据等核心流程。Worker线程整体是同步的，其中Parser\\Router\\Rewriter\\RateLimter是纯CPU计算，Executor Engine、ResultManager底层基于JDBC规范，JDBC目前对外暴露的是一个同步的调用模型，在数据库未响应前线程状态为blocked，因此Worker线程池的配置决定了Proxy整体的并发能力。 线程资源在操作系统中是相对昂贵的资源，在CPU核心数固定的情况下，线程数量过大会耗费大量内存和导致上下文频繁切换，降低性能。我们在压测过程中额外关注了线程数对服务的影响，找到了一个合适的计算公式。\n\n```Plaintext\nMath.min(cpuNum * X, maxThreadsThreshold)\n```\n\n- X：默认75，可配置\n- maxThreadsThreshold：默认800，可配置\n\n同时我们自研了DynamicBlockingQueue，在队列积压任务到一定阈值后提前创建新线程执行任务。\n\n**3.3 连接模型**\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=NmExYWRlYzEzOTBiYmQ0N2MyZTE1YzAyYWYzOWQ3MWJfZ2oyeHM0R3FVM2lqaUEwMk9wc3E4R0p4RW53UDB4bTBfVG9rZW46Ym94Y253YVNQeWpFZUxOM2NiZGRxTEplMzRiXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n在接受客户端连接后，Proxy内部会为每个客户端连接维护1个Backend Connection的逻辑连接。Backend Connection的connections列表保存执行过程中用到的真实连接，请求执行完毕或事务结束后清空connections列表，释放物理连接。同时，Proxy内部使用连接池技术来保存物理连接，减少连接建立时间并控制总的连接数量\n\n**3.4 事务模型**接下来聊一下在事务场景中的执行流程。\n\n**3.4.1 单库事务**假设有以下事务场景：\n\n```Plaintext\nbegin;\nselect * from t_order where order_id = 1;\nupdate t_order set gmt_modify = now() where order_id = 1;\nupdate t_order_item set cnt = 2 where order_id = 1;\ncommit/rollback;\n```\n\n事务过程中一共会和数据库顺序交互5次。 语句1：begin时Proxy并不知道后面要执行什么语句，无法路由到RDS，只是在连接上记录了一个begin状态语句2：执行带分片键的select，还是以16分片为例，最终语句会路由到t_order_1这张表中，获取connection_db1，先执行begin，再执行select，并放入逻辑连接的连接列表中\n\n> connection_db1为t_order_1表所在数据库上的连接\n\n语句3、语句4：路由到t_order_1,t_order_item_1,和语句2路由到相同DB，复用connection_db1执行语句5：单纯的commit或rollback语句，取出当前逻辑连接的所有RDS连接，挨个执行commit/rollback在事务执行过程中，是否会产生分布式事务完全由用户SQL语句来控制，如果事务执行过程中不会切换数据库，则会退化成单纯的RDS transaction，保持完整的ACID特性，如果事务执行过程中出现路由到多个DB的情况，则会产生分布式事务。\n\n**3.4.2 分布式事务**目前分布式事务的解决方案主要有以下3种\n\n**（1）最终一致**\n\n业务发起的最终一致性方案，如Seata\\TCC等，业务有感，多用于跨服务调用场景，如订单和库存系统，某一环提交失败需特定逻辑来整体回滚，不适合Proxy场景\n\n**（2）强一致XA**\n\n多用于跨服务调用场景，目前存在性能不佳，协调器单点\\Recovery锁占用等问题，不适合高并发场景使用\n\n**（3）厂商提供分布式事务**\n\n分布式数据库厂商如OceanBase本身会对数据进行分片，事务执行过程中操作了多个分布式节点从而引入分布式事务。针对单个分布式库内的事务保证ACID，Proxy后端基于MySQL协议，无法在多个RDS间实现这类事务。\n\n**（4）现状**\n\nCDAS Proxy并没有提供XA来保证跨RDS实例的强一致性，也没有在内部支持最终一致性。如果触发多库事务则分开提交，有部分提交成功、部分失败的风险，因此建议业务方在设计事务时尽量不再RDS集群内部产生跨库事务。\n\n**3.5 分片模型**\n\n以4库16分片场景为例阐述CDAS分片方案和传统方案的区别\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=M2QzMmRkNGU3YWQ4ZjQxOTM0MTM5NDRmZmZiNTZkMjlfc3ZiM2Y0akdiVnBKNjZHTUgyZ1F1bG9rOTJubkR5cmxfVG9rZW46Ym94Y25TWU1xTnhaTFhvaFlhSTZFWXo1Q2xjXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n传统方案每个DB中的分片名都是从后缀0开始的，每个库中有相同的分片数量。有以下劣势：\n\n- 热点分片处理困难 \n- 无法针对分片进行迁移 \n- 分片名无意义，无法自解释 \n\n针对以上问题，我们采用了类似范围分片的思路来优化了分片名。\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ZGJkYTA5NDM3YjRmOTA3YzUzMjRjZTY2MWMyNGVjMjRfWk1uaGFVdG5mdWd3QWFEc2wxazVSeVhWSFVZak52ZXFfVG9rZW46Ym94Y253NEJRTHMxejFNVDFTNXQzM2o1SkZjXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n分片名是唯一的，位于不同DB中的分片名互不相同，这种设计方式与Redis/MongoDB中不同slot位于不同实例上很相似，有利于扩展。\n\n| 问题         | 解决方式                                                     |\n| ------------ | ------------------------------------------------------------ |\n| 热点分片     | 热点分片可以通过扩展1个新RDS来解决，如添加DB5，将某个流量特别大的热点迁移到DB5 |\n| 分片迁移     | 单个分片迁移不影响其他分片，也不会被分片算法影响             |\n| 分片名无意义 | 通过分片名唯一让每个分片能表述它在整体集合中的位置           |\n\n t_order_1为热点分片或DB1压力过大时迁移t_order_1到新的DB5中：\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=Mjg3NDVkZTE5ZDk1ZDViM2ZlZjQzN2RiOGZmYmViMGJfWk1NNEdxTXVBNmhsZGpwdEtZQ0hDZ095dTFoR3JJemhfVG9rZW46Ym94Y25rbDJLQ1dNTXB2RVdHS1Z0cTF1aWpmXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n当然这种迁移方式并不适用于所有场景，如表t_order、t_order_item的分片算法完全相同，则他们一定落到同一个DB中，如果迁移过程中只迁移其中一张表势必会导致不必要的分布式事务，因此迁移时会将绑定关系的表一并迁移。**3.6 执行流程** 以最简单的场景举例，逻辑表：t_order分16片，分片键：order_id\n\n```Plaintext\nselect * from t_order where order_id=10\n```\n\n**（1）解析**\n\nWorker线程拿到这条语句后首先要做的是解析SQL语句，基于开源产品Antlr实现语法树解析。\n\n```Plaintext\n{type：select，table：t_order，condition: order_id=10}\n```\n\n**（2）路由**\n\n在获取到分片表名和条件后，根据表名获取分片规则并计算分片\n\n```Plaintext\norder_id mod 16 => 10 mod 16 => 10\n```\n\n得到真实的table：**t_order_10**（**3）重写**重写流程会将逻辑表名替换为真实表名，并计算真实位于哪个DB中。\n\n```Plaintext\nselect * from t_order_10 where order_id=10\n```\n\n**（4）限流**\n\n限流功能可以控制某类SQL的瞬时并发度，目前仅对部分场景进行了限流，如OLAP请求。\n\n**（5）执行**\n\n语句将会发往真实的数据库中执行，目前OLTP请求使用MySQL官方Connector/J发送，OLAP请求使用ClickHouse官方SDK发送，并且AP流量不会路由+重写，因为在整体设计上ClickHouse集群中的分布式表名和逻辑表名是相同的\n\n**（6）聚合结果**\n\n聚合的主要场景是查询请求被路由到了超过一个真实表的场景，如\n\n```Plaintext\nselect * from t_order where order_id in (10,11)\n```\n\n这种情况一条SQL最终到DB层面会是两次子请求\n\n```Plaintext\nselect * from t_order_10 where order_id in (10,11)select * from t_order_11 where order_id in (10,11)\n```\n\n聚合时，通过JDBC API遍历两个子请求的ResultSet，在内存中新建一个逻辑ResultSet，当逻辑ResultSet被遍历时会触发子ResultSet被遍历，最终将逻辑ResultSet编码在MySQL协议包发往客户端。如果SQL语句中包含order by，聚合时会在内存里面进行归并。内存里面排序需要将所有行数据加载到内存中排序，内存压力会很大，因此虽然有排序能力，我们还是不建议在请求分裂多个子请求的场景中使用order by语法。\n\n**3.7 HTAP**HTAP的是指混合OLTP（Online Transactional Processing）和 OLAP（Online Analytical Processing），在数据库中同时支持在线事务请求和后台分析请求。 MySQL是一个典型的OLTP型数据库，不太适合OLAP型业务，因为分析语句一般需要执行复杂的聚合和查询大量数据，这可能会影响其在OLTP上的稳定性。同时OLAP数据库一般在存储方式上采用列存来最大化压缩效率、空间占用，而MySQL采用行存，一般情况下须先将整行记录查询出来再过滤出某些列，在执行分析请求时会有明显的IO读放大，效率不高。CDAS通过DTS（数据传输）服务将MySQL各分片数据传输并聚合到ClickHouse（分析型列存数据库），形成统一的数据视图。AP流量到达Proxy后会被转发到ClickHouse，数据响应后再以MySQL数据包的形式返回给客户端，通过这种方式，CDAS实现了基于MySQL协议的HTAP能力。\n\n## **4.部署架构**\n\n部署拓扑图如下：\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MWNiOGQ1YzAxODA2OTMxYzdhZGYwOWY1OTQzNTBhZmVfMGhLaXF6c1FsTWUyQ2hFbFhGZTYwc3Vmcnd3R0JSUHJfVG9rZW46Ym94Y25acFNIek5pMlJ6NG5BS3c1bFNSd2I4XzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n现代架构中业务方对高可用的要求越来越高，目前大多数重要的服务都会采用多机房主备或互为主备的方案来部署，一般采用双机房或双中心化方案，保证其中一个区域挂掉后仍然能对外服务。在设计之初CDAS考虑到作为L7层的数据流量入口拥有极高的高可用要求，因此允许用户申请双机房Proxy集群，保证其中1个机房挂掉后另1个机房依然可以承接流量对外服务，实现机房级的高可用。业务系统通过L4负载均衡来和Proxy建立连接，L4和Proxy紧密协调保证优雅下线以及新实例发现，并感知Proxy节点的健康状态，及时下线不可用节点。同时由L4来做同机房优先路由、流量转发，Proxy后的所有MySQL集群都采用semi-sync的高可用架构模式。这种部署架构使CDAS具备了机房级灾备的能力。\n\n## **5.性能测试**\n\n在测试过程中我们尤其关注了性能方面的损耗，并对Proxy代理、直连RDS进行了对比测试，这样就能直观的得到性能损耗程度。\n\n**5.1 压测工具**\n内部压测平台(内部基于JMeter)\n\n**5.2 压测方式**\nJava SDK + 连接池方式压测 主要分为两组\n\n- 压测Proxy，Proxy连接2个RDS集群  \n- 压测单个RDS集群，作为Proxy性能对比的基准 \n\n压测语句：参考**SysBench**的数据库压测语句，由于分片场景一般不会直接使用主键ID分片，且主键ID在InnoDB中查询效率高于二级索引，因此添加sharding_k列进行分片测试，更符合实际的应用场景。\n\n```Plaintext\nCREATE TABLE `sbctest_x` (  \n`id` bigint(11) NOT NULL,\n`sharding_k` bigint(11) NOT NULL,\n`k` int(11) DEFAULT NOT NULL,  \n`name` varchar(100) NOT NULL,  \n`ts` timestamp NULL DEFAULT NULL,  \n`dt` datetime DEFAULT NULL,  \n`c` char(100) DEFAULT NULL,  \n`f` float DEFAULT NULL,  \n`t` tinyint(4) DEFAULT NULL,  \n`s` smallint(6) DEFAULT NULL,  \nPRIMARY KEY (`id`),  \nKEY `sharding_k_idx` (`sharding_k`),\nKEY `k_idx` (`k`),\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n\n压测SQL\n#QPS 非事务的简单查询\nSELECT COUNT(1) FROM $table WHERE sharding_k=?\n\n#TPS 包含 1条查询，4条变更语句\nBEGIN;\nSELECT c FROM $table WHERE sharding_k=?\nUPDATE $table SET k=k+1 WHERE sharding_k=?\nUPDATE $table SET c=?,ts=?,dt=? WHERE sharding_k=?\nDELETE FROM $table WHERE sharding_k=?\nINSERT INTO $table (id,sharding_k,k,name,ts,dt,c,pad,f,t,s) VALUES(?,?,?,?,?,?,?,?,?,?,?)\nCOMMIT;\n```\n\n**5.3 压测报告** 以8C16G规格为例：SSD\\100分片\\单片250W行\\数据量大于InnoDB buffer pool\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MjczNGU0ZWI0OTczM2I0NzRmNDQxMGJiY2UxMTY2OTlfMFZEQWlLd3BwT1V3aDNaUEt0UjBheGZYTFJjMXVidlZfVG9rZW46Ym94Y254UzQxVW8zM0NsSlZwbzZFOFo0VmFkXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n**（1） QPS 场景**\nProxy QPS只有RDS QPS的1/2，压测过程中观察到RT约为2倍 QPS场景主要是大量非事务查询的场景，Proxy的性能损耗约1/2，QPS压测过程中我们观察到Proxy后的RDS实例CPU使用率较低，于是将Proxy的CPU规格提升1倍后，Proxy QPS立即提升1倍，接近直连RDS的性能。\n**（2） TPS 场景**\n由于TPS涉及事务提交、log等操作，I/O频繁，瓶颈也主要在I/O，即使是SSD磁盘性能也都不高。Proxy后挂载了2个RDS实例，TPS在各并发用户数中都明显高于直连单个RDS，且在并发用户数增加后可达到其2倍，性能损耗几乎可忽略。\n**（3） 结论**\n如果业务场景中非事务查询的请求占绝大多数，且RDS的CPU利用率较高，建议选择Proxy的规格时CPU要高于RDS的规格如果业务场景中事务执行请求较多，Proxy不会成为性能瓶颈，规格可和RDS保持一致\n\n## **6.案例**\n\n目前支持了多个在线业务，拥有完善的指标展示、报警机制，我们例举个业务的控制台界面及监控图。 Proxy列表：\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MWU2YjRmMWFmMDNiOGI2OTkwYjI3NWY0MmZmMzQzMTVfUnd2aGpZNVdjT3Jvck5pOEYzQXJXcDE5TFczM1pwd1BfVG9rZW46Ym94Y25uMVpueEg5emlkVUx5eWQ4d1lxeWtkXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n单个节点TPS:\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=NzYxODMxN2ZlNDRmMDNiMTI2MzliMTMyMjFlYzllMmFfb0djVHU2dThteDZueGxTQW53dlZaRG1RQ0sxOXluSjNfVG9rZW46Ym94Y25DS2JjNjFBY2cxMVRCaE5hM1BmS1ZoXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n单个节点QPS:\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ODg3ZDg2NGZkMmVkNjkzMWExYmUwMWQ1YmZkMTkzZTZfcnlwc3NKTFgyQW1MYlM5MGRZakc1dGxFdXZVZnVYVGhfVG9rZW46Ym94Y24ycEhPSzA0RGc4RThtN1ZJR1hrVU9nXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n在多个产品的使用过程中CDAS表现稳定，我们将会继续往更大流量的场景推广，服务更多的业务方。 \n\n## **7.总结与展望**\n\nCDAS为数据分片而生，构建基于MySQL分片场景的内部标准，通过MySQL协议实现跨语言可访问，统一使用方式，简化使用成本。  未来我们将持续专注于CDAS的性能提升和功能补充，主要体现在下面2个方面： \n\n1. ShardingSphere社区已计划将Proxy的数据库驱动从MySQL Connnector/J 替换为事件驱动组件vertx-mysql-client（一款Reactive MySQL Client），基于Netty实现和MySQL交互I/O多路复用、事件驱动替换后worker线程池执行语句时将从blocking等待变为事件驱动，少量的线程即可支持大量的并发，我们将持续关注社区的开发进度、积极参与。 \n\n2. CDAS和MySQL交互过程中会自动解码Rows数据包，产生大量的对象并且都是朝生夕灭的，导致GC压力增大，同时编解码也会耗费大量CPU时间。 Rows数据包在非解密等特殊场景外都可以基于Netty ByteBuf实现零拷贝发送，这是最佳的方案。我们计划在开源版本替换为vertx-mysql-client后对这部分逻辑进行优化，使绝大多数的请求的响应时间能达到接近4层负载均衡的性能。","date":"2022-03-14","author":"SphereEx","excerpt":null,"metaDescription":null,"createdAt":"2023-03-30T07:01:59.275Z","updatedAt":"2023-03-30T07:02:00.368Z","publishedAt":"2023-03-30T07:02:00.365Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":12,"attributes":{"feature":true,"title":"增强企业数据服务及管控能力｜SphereEx-Console 可视化数据管控平台正式发布","content":">**1 月 27 日，SphereEx 正式推出可视化数据服务管控平台：SphereEx-Console。**\n>\n>作为一款以管理 ShardingSphere 为核心的 SphereEx 商业数据服务集群平台，SphereEx-Console 以平台的形式简化用户配置，极大简化用户操作和部署流程，进而降低新手入门门槛和企业管控难度，进一步提升企业运维效率。\n\n随着数据体量增长以及前端业务敏捷性的要求，金融、政府、传统制造等领域均面临着提升底层数据架构适配性、面向分布式生态进行架构升级。但传统集中式数据库在海量数据承载能力方面存在先天性不足，于是近些年来分布式数据库不断涌现，很好地满足这一诉求。但我们也看到，分布式架构天然的复杂性，对使用者提出了更高的要求，如何在新架构下简化用户体验，便捷普惠地享受到新架构带来的优势，成为各家重点考虑的因素。\n\n作为近些年火热的开源项目，Apache ShardingSphere 的应用范围越来越广，越来越多的企业选择采用 “数据库 + ShardingSphere” 的方式来实现传统关系型数据库到分布式数据库的转型过渡。以 Apache ShardingSphere 为基础，在原有数据库之上构建一套数据中间平台，将分布式数据服务增强能力部署在中间层，而不需对底层数据架构做出调整，进而帮助企业以更加平滑的方式获得分布式以及云上迁移能力。\n\n因此，为进一步降低用户应用门槛，拓宽 Apache ShardingSphere 的使用范围，**SphereEx 在开源内核 Apache ShardingSphere 的基础上推出 SphereEx-Console 可视化数据管控平台。** 作为一款以管理 ShardingSphere 为核心的 SphereEx 商业数据服务集群平台，SphereEx-Console 以平台的形式简化用户配置，极大简化用户操作和部署流程，进而降低新手入门门槛和企业管控难度，进一步提升企业运维效率。\n\n## 什么是 SphereEx-Console\n\n**SphereEx 控制台主要用于对 SphereEx 数据平台的运维管控，在互联网、金融、政务行业均能够支持运维和开发人员常见的操作场景。**\n\n作为一款以管理 ShardingSphere 为核心的 SphereEx 商业数据服务集群平台，SphereEx-Console 可视化数据管控平台以简便易用的操作界面代替相对复杂代码输入方式，结合内部针对资源、实例、插件等多种功能封装，为用户带来一站式的使用体验。\n\n![1.png](https://official-media.sphere-ex.com/1_5bd2d7ea7f.png)\n\n### 产品优势\n\n**功能易用**\n\n提高了用户体验，避免了配置错误，用户不需要通过配置和命令操作 SphereEx 企业数据服务平台，易用性得到极大提升，实现了平台使用的“零”瓶颈。\n\n**功能全面**\n\n完善开源版本的管理能力，提供了对 SphereEx 企业数据服务平台从基础资源到插件能力的管控，形成了对周边插件实现统一管控的解决方案。\n\n**功能可视**\n\n原生提供可视化的监控数据面板，可以实时在线查看 SphereEx 企业数据服务平台的集群、实例、主机的监控数据。\n\n\n### 行业价值\n\n**本次 SphereEx-Console 的发布，是 SphereEx 在数据管理生态下一次关键尝试，弥补了 Apache ShardingSphere 过去在管理易用性、可视化界面等维度的不足。** SphereEx-Console 以可视的操作界面，极大降低了企业运维人员的学习成本；统一的数据管控平台，也极大提升了企业的运维管理效率。\n\n作为一家以开源项目 Apache ShardingSphere 为基本的商业化公司，SphereEx 将持续增强自身在数据治理、管控等方面的能力，配以周边完善的生态基础，为用户提供更加全面和完善的 Apache ShardingSphere 应用生态以及 SphereEx 产品矩阵能力。\n\n**[点击了解详情](https://sphere-ex.com/zh/products/console/)**","date":"2022-01-26","author":"SphereEx","excerpt":"提供企业级的平台管控能力","metaDescription":null,"createdAt":"2022-03-11T09:39:49.579Z","updatedAt":"2022-03-11T09:55:15.098Z","publishedAt":"2022-03-11T09:40:24.284Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":{"id":177,"attributes":{"name":"SphereEx-Console_画板 1.png","alternativeText":"SphereEx-Console_画板 1.png","caption":"SphereEx-Console_画板 1.png","width":3750,"height":1596,"formats":{"thumbnail":{"name":"thumbnail_SphereEx-Console_画板 1.png","hash":"thumbnail_Sphere_Ex_Console_1_3552d392b3","ext":".png","mime":"image/png","width":245,"height":104,"size":26.11,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_Sphere_Ex_Console_1_3552d392b3.png"},"large":{"name":"large_SphereEx-Console_画板 1.png","hash":"large_Sphere_Ex_Console_1_3552d392b3","ext":".png","mime":"image/png","width":1000,"height":426,"size":185.42,"path":null,"url":"https://official-media.sphere-ex.com/large_Sphere_Ex_Console_1_3552d392b3.png"},"medium":{"name":"medium_SphereEx-Console_画板 1.png","hash":"medium_Sphere_Ex_Console_1_3552d392b3","ext":".png","mime":"image/png","width":750,"height":319,"size":128.74,"path":null,"url":"https://official-media.sphere-ex.com/medium_Sphere_Ex_Console_1_3552d392b3.png"},"small":{"name":"small_SphereEx-Console_画板 1.png","hash":"small_Sphere_Ex_Console_1_3552d392b3","ext":".png","mime":"image/png","width":500,"height":213,"size":73.43,"path":null,"url":"https://official-media.sphere-ex.com/small_Sphere_Ex_Console_1_3552d392b3.png"}},"hash":"Sphere_Ex_Console_1_3552d392b3","ext":".png","mime":"image/png","size":345.65,"url":"https://official-media.sphere-ex.com/Sphere_Ex_Console_1_3552d392b3.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-03-11T09:27:36.623Z","updatedAt":"2022-03-11T09:27:36.623Z"}}},"metaLogo":{"data":null},"localizations":{"data":[{"id":11,"attributes":{"feature":false,"title":"The SphereEx-Console Enterprise Grade Data Governance Platform is Officially Released","content":">On January 27th, SphereEx officially launched SphereEx-Console, a data monitoring and management platform. It provides a one-stop solution incorporating Apache ShardingSphere and SphereEx as the core, a powerful user interface to manage data clusters, and a platform to simplify user configuration.\n>\n>By simplifying operation & deployment processes, reducing entry barriers and data management challenges, SphereEx-Console improves enterprise operations and maintenance efficiency. \n\n\nExpanding data volumes and increasingly demanding agility requirements from front-end businesses have created significant challenges for enterprises in the finance, government and traditional manufacturing sectors. \n\nImproving underlying data architectures' adaptability, and upgrading it to a distributed ecosystem represent some of the new requirements - however, traditional centralized databases are inherently inadequate in terms of massive data carrying capacity. \n\nAs a result, distributed databases have emerged in recent years to meet this demand. Nevertheless, the complexity of distributed architectures poses great challenges to users. Simplifying users' experience has become a key database industry driver.\n\nAs a popular open source project, Apache ShardingSphere has successfully been implemented in multiple production scenarios, with an increasing number of enterprises adopting the Database Plus approach to transition from traditional relational databases to distributed databases. \n\nBased on Apache ShardingSphere a data platform on top of the existing databases is built, enabling distributed data capability to be deployed in the middle layer without requiring adjustments to the underlying data architecture. This approach allows enterprises to smoothly transition to distributed and cloud based platforms.\n\nThe SphereEx Enterprise Data Service Platform was created by leveraging ShardingSphere's open source kernel, and enhancing it with enterprise grade features. It provides improved data service capabilities, such as data sharding, data security etc.\n\n**SphereEx-Console is a powerful dashboard applied to the management and control of the SphereEx Enterprise Data Service Platform. It provides a one-stop solution with a user-friendly experience packaging resources, instances and plugins.**\n\nIt provides a visual approach to manage data clusters with Apache ShardingSphere and SphereEx as the core, while offering a platform to simplify user configuration, and ultimately improve enterprise operations & maintenance efficiency.\n\n## What is SphereEx-Console?\n\nThe SphereEx Enterprise Data Service Platform was created by leveraging ShardnginSphere's open source kernel, and enhancing it with enterprise grade features. It provides improved data service capabilities, such as data sharding, data security etc.\n\nSphereEx-Console is a powerful dashboard applied to the management and control of the SphereEx Enterprise Data Service Platform. It provides a one-stop solution with a user-friendly experience packaging multiple functions such as resource layer, instance layer and application layer.\n\nThe SphereEx-Console is used to manage the operation and maintenance of SphereEx Enterprise Data Service Platform. It supports operation, maintenance and development scenarios in Internet, finance and government industries.\n\nAs a SphereEx business data services cluster platform with ShardingSphere at its core, the SphereEx-Console visual data control platform provides a one-stop solution with a user friendly interface instead of complex code input, packaged with various features for the resource, instance and application layers. \n\n![Diagram Console EN.png](https://official-media.sphere-ex.com/Diagram_Console_EN_5f98720b3f.png)\n\n\n### SphereEx-Console Advantages\n\n**Easy to Use**\n\nIt improves the user experience and eliminates configuration errors. Users are not required to operate the SphereEx Enterprise Data Service Platform through configuration commands, greatly improving user experience. \n\n**Comprehensive**\n\nIt provides a unified solution for the management and control of the SphereEx Enterprise Data Service Platform - from basic resources to plugin capabilities.\n\n**Visual Monitoring**\n\nWith its data monitoring & management dashboard, SphereEx Enterprise Data Service Platform's clusters, instances and hosts monitoring data are visible online and in real time.\n\n### Industry Leading\n\nThe release of SphereEx-Console is an essential milestone for SphereEx's data management ecosystem. It improves and evolves Apache ShardingSphere in areas such as management difficulty and user interface. \n\nThe user interface greatly reduces enterprise operations & maintenance teams' learning costs, while the unified data management platform greatly improves operations and maintenance efficiency.\n\nAs a business founded the open source project Apache ShardingSphere's core team, SphereEx continues to enhance data governance as well as management & control capabilities. We are committed to providing users with a comprehensive Apache ShardingSphere application ecosystem and SphereEx enterprise grade capabilities.\n\n[Click here for more details!](https://sphere-ex.com/en/products/console/)","date":"2022-01-26","author":"SphereEx","excerpt":"Enhancing Enterprise Data Governance Capabilities","metaDescription":null,"createdAt":"2022-03-11T09:38:54.812Z","updatedAt":"2022-09-27T08:42:51.504Z","publishedAt":"2022-03-11T09:59:17.395Z","locale":"en"}}]}}},{"id":1,"attributes":{"feature":true,"title":"Apache ShardingSphere 在京东白条场景的落地之旅","content":"\n>京东白条使用 Apache ShardingSphere 解决了千亿数据存储和扩容的问题，为大促活动奠定了基础。\n>\n>2014 年初，“京东白条”作为业内互联网信用支付产品，数据量爆发式的增长，每一次大促备战都是对技术人员的考验，每一次的战略转型驱动着数据架构的成长。\n>\n>— 张栋芳，京东白条研发负责人\n\n### 项目背景\n\n自 2014 年 2 月京东白条业务上线起，为满足快速发展的业务和激增的海量数据，白条的数据架构经历了数次演进。\n\n白条数据架构的演进间接地反应了互联网消费金融的飞速发展，也说明了每一种解决方案在不同背景下都有不同的保质期。\n\n### 业务挑战\n\n随着产品升级迭代，早期的解决方案演变成为了眼前的问题，通过业务框架实现的数据分片方案导致业务代码复杂度增加、维护成本不断攀升，紧耦合的弊端原形毕露，应用每次升级都需要投入较多的精力对分片做相应调整，研发人员难以专注于业务本身。\n\n技术团队经权衡后开始考虑使用成熟的分库分表组件来承担这部分工作，让业务系统升级和架构调整不再复杂。\n\n### 解决方案\n\nShardingSphere-JDBC 是 Apache ShardingSphere 的第一款产品，它定位为轻量级 Java 框架，在 Java 的 JDBC 层提供的额外服务。它使用客户端直连数据库，以 jar 包形式提供服务，无需额外部署和依赖，可理解为增强版的 JDBC 驱动，完全兼容 JDBC 和各种 ORM 框架。\n\nShardingSphere-JDBC 的以下特点能够很好地满足白条业务场景：\n\n- 产品成熟：经数年打磨产品成熟度高，且社区活跃；\n- 性能良好：微内核、轻量化的设计，性能损耗极小；\n- 改造量小：支持原生的 MySQL 协议，研发工作量小；\n- 扩展灵活：搭配使用迁移同步组件轻松实现数据扩展。\n\n经内部大量系统性验证之后，Apache ShardingSphere 成为了京东白条数据分片中间件的首选方案，2018 年底正式开始对接。\n\n![jd_zh_1.png](https://official-media.sphere-ex.com/jd_zh_1_fac28cd6ec.png)\n\n### 用户收益\n\n- 简化升级路径\n\n通过架构解耦，业务系统升级所涉及技术栈得到有效缩短，研发团队不再需要关注分表设计，精力全部聚焦于业务本身，升级路径得到大幅度优化；\n\n- 节省研发力量\n\n引入成熟的 Apache ShardingSphere 无需重新开发分表组件，在简化业务升级路径的基础上节省了大量研发力量；\n\n- 架构灵活扩展\n\n搭配使用 Scaling 同步迁移组件从容面对“618”和“11.11”等大型活动，系统灵活扩容。\n\n**SphereEx，让数据与应用连接更简单。**\n\n","date":"2021-12-01","author":"SphereEx","excerpt":"京东白条使用 Apache ShardingSphere 解决了千亿数据存储和扩容的问题，为大促活动奠定了基础。","metaDescription":"京东科技集团是京东集团旗下专注于以技术为产业服务的业务子集团，致力于为企业、金融机构、政府等各类客户提供全价值链的技术性产品与解决方案。","createdAt":"2021-12-11T23:10:03.172Z","updatedAt":"2021-12-14T07:22:15.304Z","publishedAt":"2021-12-11T23:10:07.307Z","locale":"zh","caseStudiesType":{"data":{"id":4,"attributes":{"name":"电商","createdAt":"2021-12-11T23:02:39.013Z","updatedAt":"2021-12-11T23:02:41.225Z","publishedAt":"2021-12-11T23:02:41.156Z","locale":"zh"}}},"cover":{"data":{"id":6,"attributes":{"name":"jd_cover.png","alternativeText":"jd_cover.png","caption":"jd_cover.png","width":1030,"height":720,"formats":{"thumbnail":{"name":"thumbnail_jd_cover.png","hash":"thumbnail_jd_cover_38f39465dd","ext":".png","mime":"image/png","width":223,"height":156,"size":9.38,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_jd_cover_38f39465dd.png"},"large":{"name":"large_jd_cover.png","hash":"large_jd_cover_38f39465dd","ext":".png","mime":"image/png","width":1000,"height":699,"size":74.1,"path":null,"url":"https://official-media.sphere-ex.com/large_jd_cover_38f39465dd.png"},"medium":{"name":"medium_jd_cover.png","hash":"medium_jd_cover_38f39465dd","ext":".png","mime":"image/png","width":750,"height":524,"size":47.51,"path":null,"url":"https://official-media.sphere-ex.com/medium_jd_cover_38f39465dd.png"},"small":{"name":"small_jd_cover.png","hash":"small_jd_cover_38f39465dd","ext":".png","mime":"image/png","width":500,"height":350,"size":25.77,"path":null,"url":"https://official-media.sphere-ex.com/small_jd_cover_38f39465dd.png"}},"hash":"jd_cover_38f39465dd","ext":".png","mime":"image/png","size":46.19,"url":"https://official-media.sphere-ex.com/jd_cover_38f39465dd.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2021-12-12T12:04:05.759Z","updatedAt":"2021-12-14T07:19:51.932Z"}}},"metaLogo":{"data":{"id":10,"attributes":{"name":"jd_logo.png","alternativeText":"jd_logo.png","caption":"jd_logo.png","width":300,"height":44,"formats":{"thumbnail":{"name":"thumbnail_jd_logo.png","hash":"thumbnail_jd_logo_591b2e5e14","ext":".png","mime":"image/png","width":245,"height":36,"size":6.43,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_jd_logo_591b2e5e14.png"}},"hash":"jd_logo_591b2e5e14","ext":".png","mime":"image/png","size":2.36,"url":"https://official-media.sphere-ex.com/jd_logo_591b2e5e14.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2021-12-12T12:04:06.233Z","updatedAt":"2021-12-14T07:19:09.532Z"}}},"localizations":{"data":[{"id":6,"attributes":{"feature":true,"title":"Apache ShardingSphere & JD Baitiao: Story of an Implementation Journey","content":">Apache ShardingSphere has become the best solution for JD Baitiao’s very large storage scenarios and scale-up. JD’s shopping festival weighed very much on our big data storage. I mean, like hundreds of millions of data sets.\n>\n>JD Baitiao, an internet credit products provider launched in 2014, has always been challenged by data explosion. Every shopping festival is like a technical test for us, but every time the technical staff professionals take proactive strategic initiatives in data architecture development to ensure everything goes smoothly.\n>\n>— Dongfang Zhang，JD Batiao R&D Director\n\n## The Baitiao Data Architecture Evolution\n\nLaunched in Feb. 2014, JD Baitiao’s data architecture has already evolved for several times because of rapid increase in data volume and rapid business growth.\n\n### From 2014 to 2015\n\nBaitiao first adopted the Solr + HBase solution to connect core business systems and other systems with key databases. Solr indexing is able to achieve fast search responses and HBase is used to store a massive amount of data.\n\n- A Solr cluster can help the core database reduce reads and writes stress.\n- However, Solr is not perfect: it has unsatisfying extensibility and intrusions.\n\n### From 2015 to 2016\n\nThe company chose NoSQL to store monthly data tables in Mongo DB clusters, temporarily meeting the demand for importing/exporting massive data in payment settlement scenarios.\n\n- The change made it quite efficient to query hot data. In addition, thanks to such unstructured data storage, staff can easily modify table structures.\n- Nevertheless, the plan was still problematic: bad extensibility, intrusions, and memory hog.\n\n### From 2016 to 2017\n\nGrowing business and growing data. Tens of billions of data volume imposed more performance and volume pressure on MongoDB. JD Baitiao’s Big Data Platform used DBRep and MySQL Slave to capture data changes and store the information data in its message center, to later be written to ES and HBase.\n\n- This plan was actually much better. It focused on real-time data and improved extensibility.\n- However, Baitiao faced high costs to maintain its code because of the data sharding problem in its architecture.\n\nThe architecture evolution of JD Baitiao is just an illustration of rapidly growing Internet consumer finance. All the methods Baitiao adopted in the past were not comprehensive, indicating that every solution would be outdated soon.\n\n## Time to Adopt a Decoupled Architecture\n\nTo ensure good system performance even with rapidly increasing data volumes, the technical team first chose the sharding architecture pattern that not only guaranteed excellent performance, but also kept code under control.\n\n### The data striping plan was actually based on application architecture:\n\nThe product continuously developed but the early solution had now become one of the biggest problems. The old data sharding plan makes code even more complex and increases maintenance expenditure. Developers have to spend much time adjusting sharding every time when the application upgrades, so they cannot concentrate on their own developments. In fact, tight coupling is to blame for the problem.\n\nTherefore, the team decided to use a mature sharding component that simplifies system upgrades and architecture changes. To compare Baitiao’s shard and ShardingSphere shard, their differences are shown in the following table.\n\n![jd_en_1.png](https://official-media.sphere-ex.com/jd_en_1_eb365b4a93.png)\n\nFor Baitiao, decoupling was the next task.\n\nApparently, JD Baitiao‘s data architecture will experience a brand-new journey to decoupling. The following three directions definitely fueled its transformation:\n\n**Concentration:** instead of database sharding built in its architecture, it applied a sharding component to place more energy on it own product development.\n\n**Easier Upgrade:** it used a decoupled architecture to simplify the R&D process of system upgrades.\n\n**Future-Oriented Plan:** it aims to improve system extensibility so that Baitiao will be more capable of confidently holding large online shopping festivals like the 618 Shopping Festival and Double 11 Shopping Festival (also known as Singles Day on November 11th).\n\nJD Baitiao is a huge business, so its business scenario is truly related to finance, high concurrency, and massive data volumes. Baitiao’s sharding component must have the following features:\n\n1. Be a mature product\n2. Have excellent performance\n3. Be able to handle big data\n4. Have an extensible architecture\n\n## The Apache ShardingSphere Solution\n\nThe lightweight Java framework ShardingSphere-JDBC is Apache ShardingSpheres’s first product providing services such as a Java Database Connectivity (JDBC) API. ShardingSphere JDBC uses jar archive file installation package and allows the client-side to directly connect to databases. So, it requires no extra deployment dependencies. It is like an enhanced JDBC driver fully compatible with JDBC and ORM frameworks.\n\nShardingSphere — JDBC provides the following features, making it the best solution in the Baitiao scenario.\n\nMature Product: ShardingSphere — JDBC is a mature product developed for years, and its open source community is very active.\n\nExcellent Performance: its micro-kernel and lightweight design hardly hinder performance.\n\nMinimum Changes: it supports native MySQL protocol and minimizes R&D workload.\n\nExtensibility: users can combine with the migration sync component to easily extend data.\n\n![jd_en_2.png](https://official-media.sphere-ex.com/jd_en_2_c92fb9e5ef.png)\n\nAfter being systematically tested for several times, Apache ShardingSphere became JD Baitiao’s first choice of data sharding middleware. The cooperation started at the end of 2018.\n\n## Product Adaptation\n\nTo provide Baitiao with better services and support the business, Apache ShardingSphere has made many improvements to its product features and performance during the implementation process. At the same time, the user case helps optimize the product in return.\n\n### Upgrade the SQL Engine\n\nBaitiao’s business logic is extremely complicated; its diversified scenario demands require the best SQL compatibility levels. Thus, Apache ShardingSphere restructures the SQL parser module to support more SQLs.\n\n- Single data node routing: 100% SQL compatibility\n- Multi data node routing：comprehensively support DML, DDL, DCL, TCL and partly DAL. Support features such as pagination, deduplication, ranking, grouping, aggregation and correlated query.\n\n### Distributed Key\n\nApache ShardingSphere provides built-in distributed key generators, such as UUID, and SNOWFLAKE. It also provides its distributed key generation API so that users can develop custom key generation algorithms to meet their special needs.\n\n### Shard Key Value Injection\n\nIf a SQL has no shard condition, Apache ShardingSphere uses ThreadLoad to manage shard key value, and users can program and add a shard condition to HintManager, making the condition only effective in the current thread. That’s so-called SQL zero intrusion.\n\nAdditionally, Apache ShardingSphere continues to optimize its other features in order to meet Baitiao’s need for high performance such as：\n\n- SQL parsing result cache\n- JDBC metadata cache\n- Bind table & Broadcast table\n- Automated execution engine & Stream merge\n\nJD Baitiao & ShardingSphere teams joint efforts amde all indicators of the product live up to their expectation. The resulting final performance is almost the same as that of a native JDBC.\n\n![jd_en_3.png](https://official-media.sphere-ex.com/jd_en_3_582e2afe89.png)\n\n## Cutover\n\nApache ShardingSphere uses its custom HASH strategy to shard data, effectively avoiding the hot data problem. The total of data nodes almost reaches ten thousand. The cutover process lasts for about 4 weeks.\n\n1. After DBRep reads data, Apache ShardingSphere synchronizes the data to the target database cluster.\n2. Two clusters run together. After a data migration, Baitiao uses its own tool to verify business and data.\n\nDBRep is fundamental to the product design of ShardingSphere-Scaling. The automation feature of Scaling definitely facilitates migration and scale-up.\n\n## Apache ShardingSphere Brings Many Benefits\n\n### Simplified Upgrade Path\n\nThe decoupled architecture effectively simplifies a technology stack required in a system upgrade, so the developer team no longer need to worry about table sharding. Instead, they can focus on the business itself. In brief, ShardingSphere greatly helps Baitiao optimize the upgrade path.\n\n### Save R&D Cost\n\nBaitiao directly uses the mature product Apache ShardingSphere, so they do not need to reinvent the wheel and save a lot of time and energy.\n\n### Improve Architecture Extensibility\n\nBaitiao also uses the sync migration component Scaling that makes system scale-up even more flexible. It’s the company’s secret for online shopping festival successes.\n\n## Summary\n\nThe growth of JD Baitiao’s business greatly stimulates its data architecture upgrades. This time, it chose Apache ShardingSphere to decouple its architecture and make future upgrades less complicated. Thank to this from now on, developers only have to concentrate on their own product optimization because its data architecture is extensible enough. The case is really a good example of ShardingSphere application in the consumer finance scenario.\n\nNow, there are more and more internet credit payment models. In the future, Apache ShardingSphere in collaboration with JD Technology will explore more business scenarios, and contribute finance-related technological innovations, further improving Internet finance.\n\n### About Apache ShardingSphere\n\nShardingSphere is one of the Apache Software Foundation Top-Level Open Source projects, used by over 170 enterprises worldwide, across various verticals such as finance, e-commerce, cloud services, tourism, logistics, education, and entertainment. Its GitHub community has so far cumulated over 14,000 stars.\n\nWe welcome more technical professionals to write articles and share their experiences and thoughts. If you are interested, feel free to contact us:\n\nShardingSphere Github: https://github.com/apache/shardingsphere\n\nShardingSphere Twitter: https://twitter.com/ShardingSphere\n\nShardingSphere Slack Channel: apacheshardingsphere.slack.com","date":"2021-12-01","author":"SphereEx","excerpt":"Apache ShardingSphere has become the best solution for JD Baitiao’s very large storage scenarios and scale-up. JD’s shopping festival weighed very much on our big data storage. I mean, like hundreds of millions of data sets.","metaDescription":"JD Technology Group is a business subgroup of JD Group that focuses on technology as an industry service, and is committed to providing technical products and solutions for enterprises, financial institutions, governments and other types of customers across the value chain.","createdAt":"2021-12-12T12:30:44.957Z","updatedAt":"2022-09-26T09:16:15.245Z","publishedAt":"2021-12-12T12:35:08.683Z","locale":"en"}}]}}},{"id":2,"attributes":{"feature":false,"title":"葫芦 App 后台数据处理的逻辑捷径","content":"> ShardingSphere 大大简化了分库分表的开发和维护工作，对于业务的快速上线起到了非常大的支撑作用，保守估计 ShardingSphere 至少为我们节省了 4 个月的研发成本。\n> \n> –– 史墨轩，易华录·技术总监\n\n### 项目背景\n\n伴随着易华录旗下面向个人用户的云服务产品【葫芦 App】正式上线，后台架构所承受的业务压力也与日俱增。\n\n为此，葫芦 App 研发团队选择采用 ShardingSphere 分库分表的功能对数据进行了横向的拆分，围绕 ShardingSphere 灵活敏捷的特性，满足了葫芦 App 业务对数据层扩展性的要求，避免团队重复“造轮子”，最大程度简化了随着业务增长而带来的愈发复杂化的分库分表的开发与维护工作。\n\n### 业务挑战\n\n- 扩展能力\n- 效率提升\n- 上线提前\n- 成熟技术\n\n### 解决方案\n\n- 可插拔架构的『高扩展性』\n\n由于业务特性，葫芦 App 原本有限的存储空间被消耗得非常快，并逐渐开始影响用户在前端的响应效率。通过采用 ShardingSphere 的分片策略，葫芦研发团队在应对海量计算+存储所带来的业务问题同时，能够确保分片扩展策略的灵活性。基于此，葫芦团队可以在ShardingSphere 上快速做出相应的功能扩展，为后续架构调整提供优化方案，进一步强化突出了 ShardingSphere 分库分表的优势。\n\n- 距离业务更紧密的『贴合性』\n\n后台架构的变化越小，对于业务而言就越可控。ShardingSphere 这种位于数据库之上的生态，距离业务更近，部署起来也更加轻量，无疑是解决葫芦 App 前台业务变更与后台架构调整之间矛盾的最优解。此外在灵活性层面，ShardingSphere 通过相关配置即可实现的方式，极大简化了葫芦研发团队在分库分表层面的研发与维护工作，对于业务的快速上线起到了非常大的支撑作用。\n\n- 对于业务架构的『零侵入性』\n\n葫芦 App 选择了 ShardingSphere-Proxy 部署模式，在不更换底层数据库的前提下通过 Proxy 来管理真实的数据库集群，基本无需对业务进行改造就已经完成了业务与数据在架构层面的分离，并避免了因更换数据库导致的业务不可用、漫长稳定周期等风险。另外，ShardingSphere 的无状态模式，几乎不会对前端用户产生任何可感知的影响，业务层也无需关注数据的存储方式。\n\n![ehualu_zh_1.png](https://official-media.sphere-ex.com/ehualu_zh_1_3fad8e2ccd.png)\n\n### 用户收益\n\n- 节省 4 人月成本\n- 明显改善业务体验\n- 系统灵活扩容\n\n**SphereEx，让数据与应用连接更简单。**","date":"2021-11-29","author":"SphereEx","excerpt":"ShardingSphere 大大简化了分库分表的开发和维护工作，对于业务的快速上线起到了非常大的支撑作用，保守估计 ShardingSphere 至少为我们节省了 4 个月的研发成本。","metaDescription":"北京易华录信息技术股份有限公司，成立于 2001 年，是国务院国资委直接监管的中央企业中国华录集团旗下控股的上市公司，致力于建设城市数字经济基础设施。","createdAt":"2021-12-11T23:11:13.132Z","updatedAt":"2022-11-30T08:25:32.894Z","publishedAt":"2021-12-11T23:11:16.643Z","locale":"zh","caseStudiesType":{"data":{"id":3,"attributes":{"name":"金融","createdAt":"2021-12-11T23:02:27.233Z","updatedAt":"2021-12-11T23:02:50.766Z","publishedAt":"2021-12-11T23:02:50.699Z","locale":"zh"}}},"cover":{"data":{"id":7,"attributes":{"name":"ehualu.png","alternativeText":"ehualu.png","caption":"ehualu.png","width":1030,"height":720,"formats":{"thumbnail":{"name":"thumbnail_ehualu.png","hash":"thumbnail_ehualu_6d29db12b5","ext":".png","mime":"image/png","width":223,"height":156,"size":6.16,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_ehualu_6d29db12b5.png"},"large":{"name":"large_ehualu.png","hash":"large_ehualu_6d29db12b5","ext":".png","mime":"image/png","width":1000,"height":699,"size":53.63,"path":null,"url":"https://official-media.sphere-ex.com/large_ehualu_6d29db12b5.png"},"medium":{"name":"medium_ehualu.png","hash":"medium_ehualu_6d29db12b5","ext":".png","mime":"image/png","width":750,"height":524,"size":33.7,"path":null,"url":"https://official-media.sphere-ex.com/medium_ehualu_6d29db12b5.png"},"small":{"name":"small_ehualu.png","hash":"small_ehualu_6d29db12b5","ext":".png","mime":"image/png","width":500,"height":350,"size":18.22,"path":null,"url":"https://official-media.sphere-ex.com/small_ehualu_6d29db12b5.png"}},"hash":"ehualu_6d29db12b5","ext":".png","mime":"image/png","size":24.54,"url":"https://official-media.sphere-ex.com/ehualu_6d29db12b5.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2021-12-12T12:04:05.768Z","updatedAt":"2021-12-14T07:19:57.408Z"}}},"metaLogo":{"data":{"id":3,"attributes":{"name":"ehualu_logo.png","alternativeText":"ehualu_logo.png","caption":"ehualu_logo.png","width":303,"height":153,"formats":{"thumbnail":{"name":"thumbnail_ehualu_logo.png","hash":"thumbnail_ehualu_logo_26122a7add","ext":".png","mime":"image/png","width":245,"height":124,"size":22.6,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_ehualu_logo_26122a7add.png"}},"hash":"ehualu_logo_26122a7add","ext":".png","mime":"image/png","size":22.6,"url":"https://official-media.sphere-ex.com/ehualu_logo_26122a7add.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2021-12-12T12:04:05.036Z","updatedAt":"2021-12-14T07:19:30.734Z"}}},"localizations":{"data":[{"id":7,"attributes":{"feature":false,"title":"eHualu Story’s Data Processing Shortcut","content":"> ShardingSphere has greatly simplified sharding development and maintenance, which has played a very important role in our rapid product release. We estimate that ShardingSphere has saved us at least 4 months of R&D costs.\n> \n> –– Shi Moxuan, Technical Director，E-Hualu\n\nAt the beginning of this year, E-hualu officially launched its cloud service product “Hulu App” that targets non-business users. Since then, its architecture’s data stress has increased on a daily basis.\n\nThe R&D team of Hulu App chose to use ShardingSphere’s sharding features and scale their data out. The flexible and agile features of ShardingSphere met the requirements of Hulu App. It requires data extensibility, and needs to prevent the team from “reinventing the wheel”.\n\nShardingSphere can simplify sharding development and maintenance to the greatest extent. As the service is growing, the work will be increasingly complicated.\n\n### Hulu App’s growth pressure: feature extension and new services\n\nThe high requirement for data storage and computing is in the genes of the Hulu App technical team.\n\nBecause Hulu App is rapidly growing, its service and feature changes are relatively frequent. Therefore its technical team needs to adapt quickly when a front-end service changes. As its users and service data volume are growing rapidly, its databases are faced with greater pressure.\n\nThe official Hulu App was released on May 17, 2020, and then its user data and service volume also showed a rapid growth trend. Therefore it was inevitable to scale out its databases for many times. At the same time, with business requirements changing very quickly, new challenges have emerged:\n\n- How to ensure feature extensibility\n\nBecause of increasing users and developing product versions, its user data has exploded. The storage and computing capability of its old architecture has been severely challenged. Therefore, Hulu App requires that its back-end data processing platform shall have both extensible features, and flexibility.\n\n- How to improve efficiency\n\nFaced with its changeable services, the R&D team of Hulu App needs to make rapid adjustments when necessary, so as to make its architecture adapt to its services. A highly-flexible, extensible data architecture can greatly improve development efficiency of the team. On the other hand, because of such large amounts of data, database retrieval efficiency will inevitably have problems such as delays, slow read and write. These problems will ultimately affect user experience.\n\n- How to facilitate feature release\n\nAll new products will face problems like frequent new feature release, bringing the scheduled release date forward, when they are just released. These problems pose a great challenge to a development team. At first, Hulu’s development team planned to base on their service demands and create their own sharding plan all by themselves. However, they don’t have enough time to do that, so it’s urgent for them to choose a new sharding plan.\n\n- How to guarantee system stability\n\nA newly-added technology will greatly challenge system stability. It’s especially true for bottom-level technologies, most of which can influence platform services and probably challenge system stability. So what Hulu App needs is a data application product that does not greatly stress databases, but quickly adapts to them and that is also highly stable.\n\n### Leverage ShardingSphere’s features to Create a Flexible, Highly-Available Data Architecture Solution\n\nHulu App’s team studied their needs and spent two weeks evaluating ShardingSphere and other similar solutions. They had evaluative indicators like product feature, maturity, stability and performance. At last, they chose ShardingSphere because of its powerful features, its support and its high maturity can fully meet their needs.\n\nThe Hulu team deployed ShardingSphere above Alibaba Cloud’s RDS. Instead of adjusting ontology databases, the Hulu team prefers to conduct data governance above its databases. ShardingSphere can effectively improve Hulu’s data architecture, because it can achieve three things: high scalability brought by its extensible architecture, closely adaptive to Hulu’s service. ShardingSphere brings Hulu with obvious improvement:\n\n#### “High scalability” of the pluggable architecture\n\nDue to its service characteristics, Hulu App’s limited storage space is consumed very quickly, and the problem gradually begins to affect response efficiency at the front end. Therefore, the team can quickly develop enhanced features by using ShardingSphere, which can provide optimized solutions for its subsequent architectural adjustments. The example strengthens the ShardingSphere’s advantage in sharding.\n\n#### Closely adaptive to service\n\nThe smaller the architecture change is, the more controllable it is for developers. ShardingSphere, an ecosystem located above databases, is closer to service and it deployment is even more lightweight. So it is undoubtedly the best solution to solve the contradiction between the front-end service changes and back-end architecture adjustment of Hulu App. In addition, in terms of flexibility, ShardingSphere supports instant achievement of relevant configurations, which greatly helps the team save a lot of sharding-related work. So it plays a very important role in Hulu App’s quick releases.\n\n#### “Zero attack” to the service architecture\n\nHulu App chose ShardingSphere’s Proxy deployment mode. It can manage the real database cluster through Proxy without. replacing underlying databases. It basically completed the separation of service and data at the architectural level without any changes. So it can avoid the risks caused by database changes, such as unavailable service and long stable cycle. In addition, the non-status mode of ShardingSphere hardly has any perceptible impact on users at the front end and the service layer does not need to pay attention to the data storage method .\n\nSo, Hulu App is a product with tight go-live schedule and fast feature iteration. ShardingSphere-Proxy can reuse original databases and help the Hulu development team to develop incremental features above databases such as sharding and data encryption. In addition, ShardingSphere allows developers not to worry about database configuration and shields user perception, so it quickly builds service-oriented database direct connection, which is well separated from the system architecture level, and ensures that daily maintenance work at the database agent, such as bug repair and version update, will not affect the services.\n\nThis time, ShardingSphere collaborated with the R&D team of E-hualu’s Hulu App, and helped them overcome data volume explosion for several times. Actually, this is just a tiny example of ShardingSphere’s various application scenarios around the world.\n\n### About E-hualu & Hulu App\n\nFounded in 2001, Beijing E-hualu Information Technology Co., Ltd. is a listed company under the state-owned enterprise China Hualu Group Co., LTD. directly administrated by the State-owned Assets Supervision and Administration Commission of the State Council (SASAC). Its mission is to reduce energy consumption and the cost of long-term data management.\n\nHulu App is the first to-customer product of E-hualu.\n\n### ShardingSphere Community:\n\nShardingSphere Github: https://github.com/apache/shardingsphere\n\nShardingSphere Twitter: https://twitter.com/ShardingSphere\n\nShardingSphere Slack Channel: apacheshardingsphere.slack.com","date":"2021-11-29","author":"SphereEx","excerpt":"ShardingSphere has greatly simplified sharding development and maintenance, which has played a very important role in our rapid product release. We estimate that ShardingSphere has saved us at least 4 months of R&D costs.","metaDescription":"Founded in 2001, Beijing E-hualu Information Technology Co., Ltd. is a listed company under the state-owned enterprise China Hualu Group Co., LTD. directly administrated by the State-owned Assets Supervision and Administration Commission of the State Council (SASAC). Its mission is to reduce energy consumption and the cost of long-term data management.","createdAt":"2021-12-12T12:33:14.398Z","updatedAt":"2022-09-26T10:27:50.013Z","publishedAt":"2021-12-12T12:35:01.820Z","locale":"en"}}]}}},{"id":35,"attributes":{"feature":true,"title":"openGauss X ShardingSphere，分布式方案的另一种最佳实践","content":"Apache ShardingSphere 持续助力于 openGauss 分布式数据库能力的构建。openGauss 数据库自 2020 年 6 月开源以来，受到了业界的广泛关注，现已吸引众多伙伴、开发者参与其中，共建繁荣的数据库生态。面对如今海量数据，超高并发等诸多场景，openGauss 将目光转向于分布式解决方案，专注于解决海量数据存储、超高并发吞吐、大表瓶颈等众多难题，与 ShardingSphere 一起构建全栈开源分布式解决方案，实现 openGauss 的又一大突破。\n\n## 分布式解决方案\n![640.png](https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/640_f6ba503847.png)\n<center>图1 分布式解决方案整体框图</center>\n\nopenGauss 融合了众多开源组件，用以构建集数据水平扩展、分布式事务及治理一体化的全栈开源分布式解决方案，整体框图如图 1 所示。\n\n- 其中 ShardingSphere-Proxy 为开源分布式解决方案，具有分库、分表、分布式事务、弹性伸缩、读写分离等众多能力；\n- HAProxy 结合 Patroni 的 REST API，可以始终识别数据库的主节点，保证高可用场景，同时可实现负载均衡；\n- 每个 Patroni 高可用节点支持一主多备，每个节点使用 Paxos 协议保证数据的一致性，各个节点可以部署在相同或不同的区域，用以保证多地多中心的数据安全。\n本分布式方案运用 ShardingSphere-Proxy 强劲的分布式能力，通过 kubernetes 管理集群，prometheus 监控集群状态，从而构建全栈开源的分布式解决方案。\n\n## 产品优势\n\n1. **极致扩展能力，灵活扩缩容**\n计算与存储能力可通过水平拆分实现线性扩展，最高可达数据 6400 分片，性能随扩展准线性增长，可有效解决单表数据量膨胀问题；结合业务流量，灵活平滑进行数据节点的扩缩容，智能读写分离，实现分布式数据库的自动负载均衡。\n\n2. **丰富企业级特性**\n支持分布式存储过程、触发器，分布式事务，全密态数据加密，WDR 诊断报告，提供丰富的企业级特性。\n\n3. **一键部署，屏蔽底层依赖**\n标准化镜像确保多环境一致性交付，容器化部署，实现物理资源池化，降低对平台的依赖性，简洁高效，实现应用秒级部署。\n\n4. **超高可用，实现异地容灾**\n强有力的集群管理、运维能力，支持同城、异地、多地多中心灵活部署，基于 Paxos 协议保证数据的安全及强一致性，提供 RPO=0 的多种容灾能力。\n\n5. **开源开放，构建全栈生态**\n开源 openGauss 单机及分布式解决方案，鼓励更多伙伴、开发者共同参与其中，共建数据库的繁荣生态，打造全栈开源生态链。\n\n## 应用场景\n\n- **金融系统**\n基于多数派共识协议 Paxos，提供分布式强一致事务能力，确保分布式环境下数据的强一致能力；多地多中心，提供 RPO=0 的能力，保障金融级可靠。\n\n- **政企办公**\n提供全密态数据加密，安全可信，支持软硬件生态，保障高可用场景，为政企办公保驾护航。\n\n- **智慧电网**\n支持同城、异地、多地多中心灵活部署，根据业务的潮汐特点灵活扩缩容，助力智慧电网。","date":"2021-09-15","author":"SphereEx","excerpt":"Apache ShardingSphere 持续助力于 openGauss 分布式数据库能力的构建","metaDescription":null,"createdAt":"2023-03-28T07:02:39.490Z","updatedAt":"2023-03-28T07:10:25.644Z","publishedAt":"2023-03-28T07:10:25.641Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":36,"attributes":{"feature":true,"title":"数据为王，携程国际火车票的 ShardingSphere 之路","content":"## 业务背景\n\n随着国际火车票业务的高速发展，订单量快速增长，单数据库瓶颈层面的问题逐渐显露，常规的数据库优化已无法达到期望的效果。同时，原先的底层数据库设计，也存在一些历史遗留问题，比如存在部分无用字段、表通过自增主键关联和各个应用直连数据库等问题。\n\n为此，经过讨论后，我们决定对订单库进行分库分表，同时对订单表进行重构，进而从根本上解决这些问题。\n\n## 业务挑战\n\n目标确定后，实践起来可不轻松，出现了很多的问题和挑战。这里列举一些典型问题，大致可以分为两大类：分库分表通用问题、具体业务关联问题。\n\n**分库分表通用问题**\n\n- 如何切分，垂直分还是水平分？分片的键，如何选取？\n- 如何根据键值路由到对应库、对应表？\n- 采用什么中间件，代理方式还是中间件的方式？\n- 跨库操作等问题，如跨库事务和跨库关联？\n- 数据扩容问题，后续如何进行扩容？\n\n**具体业务关联问题**\n\n- 各个应用直连数据如何解决？\n- 如何进行平滑过渡？\n- 历史数据如何恰当迁移？\n\n## 方案选型\n\n### **如何切分**\n\n切分方式，一般分为垂直分库、垂直分表、水平分库和水平分表四种，如何选择，一般是根据自己的业务需求决定。\n\n我们的目标是要从根本上解决数据量大、单机性能问题等问题，垂直方式并不能满足需求，所以我们选取了水平分库+水平分表的切分方式。\n\n### **分片键选取**\n\n一般是根据自己的实际业务，来选择字段来作为分片的键，同时可以结合考虑数据的热点问题 、分布问题。比如订单系统，不能根据国家字段进行分片，否则可能会出现某些国家很多的订单记录，某些国家几乎没有订单记录，进而数据分布不均。相对正确的方式，比如订单类系统，可以选择订单 ID；会员系统，可以选择会员 ID。\n\n### **如何路由**\n\n选定了分片的键之后，接下来需要探讨的问题，就是如何路由到具体的数据库和具体的表。\n\n- 映射路由\n- 分组路由\n- 哈希路由\n- 分组哈希路由\n\n在实践中，我们结合了前面的几种方式，借鉴了他们的优点不足，而采用了此种方式。因为分组方式，能很方便的进行扩容，解决了数据扩容问题；哈希方式，能解决分布相对均匀，无单点数据库热点问题。\n\n## 选择 Apache ShardingSphere 的理由\n\n在产品选型过程中，考虑了多款产品，也包括公司自己实现的产品，最终选择了 ShardingSphere，理由如下。\n\n### **技术环境**\n\n- 我们团队是 Java 体系下的，对 Java 中间件有一些偏爱\n- 更偏向于轻量级组件，可以深入研究的组件\n- 可能会需要一些个性定制化\n\n### **专业程度**\n\n- 取决于中间件由哪个团队进行维护，是否是名师打造，是否是行业标杆\n- 更新迭代频率，最好是更新相对频繁，维护较积极的\n- 流行度问题，偏向于流行度广、社区活跃的中间件\n- 性能问题，性能能满足我们的要求\n\n### **使用成本**\n\n- 学习成本、入门成本和定制改造成本\n- 弱浸入性，对业务能较少浸入\n- 现有技术栈下的迁移成本，我们当前技术栈是 SSM 体系下\n\n### **运维成本**\n\n- 高可用、高稳定性\n- 减少硬件资源，不希望再单独引入一个代理中间件，还要考虑运维成本\n- 丰富的埋点、完善的监控","date":"2021-08-24","author":"瑞华，携程高级后端开发工程师","excerpt":"国际火车票业务分库分表重构实践","metaDescription":"我们是携程火车票研发团队，负责火车票业务的开发以及创新。火车票研发在多种交通线路联程联运算法、多种交通工具一站式预定、高并发方向不断地深入探索和创新，持续优化用户体验，提高效率，致力于为全球人民买全球火车票。","createdAt":"2023-03-28T07:18:10.674Z","updatedAt":"2023-03-28T07:18:12.210Z","publishedAt":"2023-03-28T07:18:12.206Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":39,"attributes":{"feature":true,"title":"客户端分片到Proxy分片，如丝般顺滑的平稳迁移","content":"client方式的优势是实现简单，只需要通过简单的配置即可完成拆分操作。\n\n在本地通过分片进行计算，得到真实的库和表进行路由，性能相对较高。\n\n不依赖于三方，没有单点故障。 \n\n \n\nclient方式的劣势是每个项目都要去管理分片，读写分离等信息，没办法统一进行管理。\n\n当需要升级的时候只能所有项目都进行升级，没办法统一升级。\n\n最难的点在于需要推动各个业务域进行升级，升级的周期较长，对业务方的压力也比较大。\n\n相对于proxy方式，client方式连接数占用的比较多，一个数据源10个连接，部署10个实例就是100个连接。\n\n \n\nproxy方式指的是部署一个独立的服务，这个服务会实现Mysql协议，应用中只需要连接这个独立的proxy服务，把它当做一个完整的独立的数据库使用即可。\n\n分库分表等规则全部由这个proxy去管理，对应用透明。\n\n \n\n彩虹桥DAL是得物基础架构组研发的proxy方式的代理服务，有了彩虹桥就可以统一管理所有数据库，可以对数据库操作做限流保护，可以做压测时的数据自动进入影子库，可以监控慢SQL等等。\n\n \n\n**接入方案**\n\n**直接修改老数据源-改成连接彩虹桥**\n\n\n\n最简单，最快速的方式就是直接将所有老的数据源改成新的代理数据源，然后重新发布就可以了。\n\n这个方案确实快，但是不够好，不够完美，不够稳定，为啥？\n\n\n\n这其实跟改接口是一样的道理，当某天加了个新的需求，你会发现直接在老接口的基础上改一下就完成了。改完后测试发现当前没问题，但是老版本的产品还在用，然后兼容出问题了。这种场景下如果你是对接口进行升级，之前是v1，现在改成v2，老的逻辑完全不动，就完美了呀。\n\n\n\n如果我们直接改成最新的方式，那么如何平滑的发布上线呢？\n\n \n\n全部发布，万一彩虹桥那边配置有问题，或者有其他的问题导致不可用，你这不就尴尬了么，当然也可以回滚历史版本。\n\n \n\n另一种方式就是灰度发布一台机器，进行测试，没问题后全部发布，有问题就不发，其实也不错。就怕过了一段时间，万一彩虹桥有Bug, 我们还是得回滚到接入彩虹桥之前的版本。\n\n \n\n**新增一套数据源 - 支持开关切换到新数据源，下线老数据源**\n\n \n\n新增数据源，支持开关切换就跟升级API版本是一样的逻辑。保持老逻辑不变，上线时还是走老的数据源，然后通过开关动态切换到新的数据源，完成上线动作。\n\n \n\n当然这边也会出现上面提到的问题，比如彩虹桥不可用之类的情况，也可以通过灰度配置的方式来测试。将开关配置灰度到某个节点上，只切这个节点的数据源，进行测试，没问题后可以扩大范围。有问题开关一关就还原到之前的逻辑了，无影响。\n\n \n\n如果用了一段时间，彩虹桥有Bug的话同样改下开关的配置即可回滚，也不用重新发布项目。\n\n \n\n此方案不足的点就是会占用一分部数据库的连接，因为老的数据源还存在。此时需要改代码去掉老的数据源，就能释放了。如果后面还要去改代码，那还能叫完美的方案么？\n\n \n\n这点我也考虑到了，还是通过配置开关来关闭老的数据源，但是这个操作得重启服务，重启后就只有一套数据源了。（**请确保以后只用彩虹桥的方式才用此开关**）\n\n \n\n**实现步骤**\n\n1. 配置2套新数据源，连接彩虹桥的地址，订单的库分为老订单库和新的分库分表库。\n2. 新增2个动态数据源，用于开关动态切换，使用AbstractRoutingDataSource管理。\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=NTk4N2JjMTE5ZWM2ZDYzNzE3YTc4ZjJiNmNkNjU4MDNfWkxpTWY5N1djdFUwWVRaQ1ExMmEwbDZnUllkd01SeWtfVG9rZW46Ym94Y25IQ25iS3Z3SEo4QVN0TjlrV0JlNFBkXzE2ODAwNzQxMDA6MTY4MDA3NzcwMF9WNA)\n\n3. 老数据源通过@ConditionalOnProperty来控制是否加载，默认加载，用于后期下线老数据源。\n\n4. 读写分离兼容client和proxy方式。\n\n5. 分片算法重写，之前用的Sharding-Jdbc3.X版本，新的彩虹桥基于5.X版本深度定制开发，在自定义算法这块有变化，目前彩虹桥的分片算法全部在彩虹桥的扩展包中，不在订单里面。\n\n \n\n**注意事项**\n\n**select last_insert_id()不支持** \n\n \n\n在insert中通过select last_insert_id()实时返回当前插入的自增ID场景需要修改，目前订单中就一个地方用到了，而且上层其实没消费这个ID, 如果后面有其他场景需要获取刚插入的ID可以手动提前获取分布式ID，然后再用这个ID存到表中。\n\n \n\n**强制走从节点查询**\n\n \n\n老的读写分离用的Sharding-Jdbc做的，正常情况下默认查询走从节点，其他走主节点。而现在订单里面默认都是走主节点，如果需要强制走从节点需要使用HintManagerHolder.clear() 清空Hint信息来实现。\n\n \n\n猜测最开始没有从节点，业务都走主节点。后面加了从节点，如果用默认的方式查询都会走从节点，但是在某些业务场景中一致性比较高，必须走主节点查询，相对于标记出哪些查询走从节点来说，默认全部走主节点更稳。然后根据业务场景再标记是否要走从节点，这样几乎不会影响老的逻辑。\n\n \n\n接入彩虹桥后，默认是彩虹桥和Sharding-Jdbc两套数据源共存的，所以在HintManagerHolder.clear() 这块也需要做兼容，同时支持彩虹桥和Sharding-Jdbc方式的强制走从节点。\n\n \n\n**第一步：兼容老代码**\n\n \n\n将所有使用HintManagerHolder.clear()改成SqlHintUtils.clear()。\n\n```java\npublic class SqlHintUtils {\n    /*      清空Hint信息     */    \n    public static void clear() {        // ShardingJdbc        \n    \tHintManagerHolder.clear();        // 彩虹桥DAL        \n    \tBifrostContext.clearNotAutoClearPrimary();    }\n}\n```\n\n \n\n**第二步：新增注解强制走从节点，比代码更优雅** \n\n \n\n以后有新的查询需要走从节点就在Dao方法上增加@ForceSlave注解即可，此注解只能作用于Dao方法上，加在其它层无效。如果Dao方法上加了注解，那么方法内所有的查询操作都将走从节点。\n\n \n\n老的clear相关的代码其实可以用注解代替，但为了保险起见还是不改变原有的方式，新的可以用注解的方式。","date":"2021-08-13","author":"SphereEx","excerpt":"随着订单数量的增多，以及大促时需要扛住比平时多N倍的流量，单库单表的瓶颈日益显现。\n需要对数据库进行水平拆分，目前订单使用的是客户端分片的方式进行拆分，采用Sharding-Jdbc框架实现。","metaDescription":null,"createdAt":"2023-03-29T07:22:28.685Z","updatedAt":"2023-03-29T07:22:29.697Z","publishedAt":"2023-03-29T07:22:29.693Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":44,"attributes":{"feature":true,"title":"vivo 全球商城：订单中心架构设计与实践","content":"## 一、背景\n\n   随着用户量级的快速增长，vivo 官方商城 v1.0 的单体架构逐渐暴露出弊端：模块愈发臃肿、开发效率低下、性能出现瓶颈、系统维护困难。\n\n   从2017年开始启动的 v2.0 架构升级，基于业务模块进行垂直的系统物理拆分，拆分出来业务线各司其职，提供服务化的能力，共同支撑主站业务。\n\n   订单模块是电商系统的交易核心，不断累积的数据即将达到单表存储瓶颈，系统难以支撑新品发布和大促活动期间的流量，服务化改造势在必行。\n\n   本文将介绍 vivo 商城 订单系统建设的过程中遇到的问题和解决方案，分享架构设计经验。\n\n## 二、系统架构\n\n   将订单模块从商城拆分出来，独立为订单系统，使用独立的数据库，为商城相关系统提供订单、支付、物流、售后等标准化服务。\n\n   系统架构如下图所示：\n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MjJiMzQwYmM3MTNhNTAxMjFjMmZmMDEwOGY4ZjExMzhfVTYyOUZLTDNOckRKeDJEUDBheGV2dWZXZG9qbGxOZWZfVG9rZW46Ym94Y251akVGY2dJQTV6YjdmVm1kbjVLcWxkXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n## 三、技术挑战\n\n### 3.1 数据量和高并发问题\n\n   首先面对的挑战来自存储系统：\n\n   **数据量问题**\n   - 随着历史订单不断累积，MySQL中订单表数据量已达千万级。\n   - 我们知道InnoDB存储引擎的存储结构是B+树，查找时间复杂度是O(log n)，因此当数据总量n变大时，检索速度必然会变慢， 不论如何加索引或者优化都无法解决，只能想办法减小单表数据量。\n   - 数据量大的解决方案有：**数据归档、分表**\n\n   **高并发问题**\n   - 商城业务处于高速发展期，下单量屡创新高，业务复杂度也在提升，应用程序对MySQL的访问量越来越高。\n   - 单机MySQL的处理能力是有限的，当压力过大时，所有请求的访问速度都会下降，甚至有可能使数据库宕机。\n   - 并发量高的解决方案有：**使用缓存、读写分离、分库**\n\n   下面对这些方案进行简单描述：\n\n   **数据归档** \n   - 订单数据具备时间属性，存在热尾效应，大部分情况下检索的都是最近的订单，而订单表里却存储了大量使用频率较低的老数据。 \n   - 那么就可以将新老数据分开存储，将历史订单移入另一张表中，并对代码中的查询模块做一些相应改动，便能有效解决数据量大的问题。\n\n   **使用缓存**\n   - 使用Redis作为MySQL的前置缓存，可以挡住大部分的查询请求，并降低响应时延。\n   - 缓存对商品系统这类与用户关系不大的系统效果特别好，但对订单系统而言，每个用户的订单数据都不一样，缓存命中率不算高，效果不是太好。\n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=NTAyNDg5ODJjOTJlMGQwYjExMzk4ZDA3NjY4ZGIyMTZfc25Wd0xRTlVHREJFeFBNZUphMHE3NzJnb1JDS2tweDVfVG9rZW46Ym94Y242eDlCb1l6UEpmWHdsaE44NWtWcHJMXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n   **读写分离**\n   - 主库负责执行数据更新请求，然后将数据变更实时同步到所有从库，用多个从库来分担查询请求。 \n   - 但订单数据的更新操作较多，下单高峰时主库的压力依然没有得到解决。且存在主从同步延迟，正常情况下延迟非常小，不超过1ms，但也会导致在某一个时刻的主从数据不一致。\n   - 那就需要对所有受影响的业务场景进行兼容处理，可能会做一些妥协，比如下单成功后先跳转到一个下单成功页，用户手动点击查看订单后才能看到这笔订单。\n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MDE2MzcxMDM5YTg0NDc2MGFhZDFmZWZiMTIxMGI0NWRfdE9leENINGxLU01XSjU3Z29UN1Vselk5dmZ2ZFNaaFZfVG9rZW46Ym94Y25IM3lpSWRvRkFkb0puWUdhOEVDc3FkXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n   **分库**\n   - 分库又包含垂直分库和水平分库。 \n   - **① 水平分库：**把同一个表的数据按一定规则拆到不同的数据库中，每个库可以放在不同的服务器上。\n   - **② 垂直分库：**按照业务将表进行分类，分布到不同的数据库上面，每个库可以放在不同的服务器上，它的核心理念是专库专用。\n\n   **分表**\n   - 分表又包含垂直分表和水平分表。\n\n   - **① 水平分表：**在同一个数据库内，把一个表的数据按一定规则拆到多个表中。\n\n   - **② 垂直分表：**将一个表按照字段分成多表，每个表存储其中一部分字段。\n\n     \n\n   我们综合考虑了改造成本、效果和对现有业务的影响，决定直接使用最后一招：**分库分表**\n\n​    \n\n### 3.2 分库分表技术选型\n\n   分库分表的技术选型主要从这几个方向考虑：\n\n   1. 客户端sdk开源方案\n   2. 中间件proxy开源方案\n   3. 公司中间件团队提供的自研框架\n   4. 自己动手造轮子\n\n​    \n\n   参考之前项目经验，并与公司中间件团队沟通后，采用了开源的 **Sharding-JDBC** 方案。现已更名为Sharding-Sphere。\n\n   - Github：https://github.com/sharding-sphere/\n   - 文档：官方文档比较粗糙，但是网上资料、源码解析、demo比较丰富\n   - 社区：活跃\n   - 特点：jar包方式提供，属于client端分片，支持xa事务\n\n​    \n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MmJhYWUwMmUzODY0YjkyZDc0MmQ0MjY4MzY4MGRjNzFfSmhFRFdMSG9CbXc0cFhPcW9IV2RTNWRsMkk0YnBSWVVfVG9rZW46Ym94Y25FaWpyOFZabXh0MWJSb3IyZnhUTmRmXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n​    \n\n#### **3.2.1 分库分表策略**\n\n   结合业务特性，选取用户标识作为分片键，通过计算用户标识的哈希值再取模来得到用户订单数据的库表编号.\n\n    假设共有n个库，每个库有m张表，\n\n   则库表编号的计算方式为：\n\n   \\- 库序号：Hash(userId) / m % n \n\n   \\- 表序号：Hash(userId) % m\n\n​    \n\n   路由过程如下图所示：\n\n​    \n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=YjdmMzI1NjViOGYxOTk5MTVjYTk1YjgzMmFjZGM3NTVfZDZLZ0VHZm1wUDZxb1VoTHM2U3Vsc0hvOFM1eEhvQTNfVG9rZW46Ym94Y25DYTZXNEdDbjZycDBiWnRqMGVPSFNlXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n​    \n\n​    \n\n#### **3.2.2 分库分表的局限性和应对方案**\n\n​    \n\n   分库分表解决了数据量和并发问题，但它会极大限制数据库的查询能力，有一些之前很简单的关联查询，在分库分表之后可能就没法实现了，那就需要单独对这些Sharding-JDBC不支持的SQL进行改写。\n\n​    \n\n   除此之外，还遇到了这些挑战：\n\n​    \n\n   **（1）全局唯一ID设计**\n\n​    \n\n   分库分表后，数据库自增主键不再全局唯一，不能作为订单号来使用，但很多内部系统间的交互接口只有订单号，没有用户标识这个分片键，如何用订单号来找到对应的库表呢？\n\n​    \n\n   原来，我们在生成订单号时，就将库表编号隐含在其中了。这样就能在没有用户标识的场景下，从订单号中获取库表编号。\n\n​    \n\n   **（2）历史订单号没有隐含库表信息**\n\n​    \n\n   用一张表单独存储历史订单号和用户标识的映射关系，随着时间推移，这些订单逐渐不在系统间交互，就慢慢不再被用到。\n\n​    \n\n   **（3）管理后台需要根据各种筛选条件，分页查询所有满足条件的订单**\n\n​    \n\n   将订单数据冗余存储在搜索引擎Elasticsearch中，仅用于后台查询。\n\n​    \n\n### 3.3 怎么做 MySQL 到 ES 的数据同步\n\n​    \n\n   上面说到为了便于管理后台的查询，我们将订单数据冗余存储在Elasticsearch中，那么，如何在MySQL的订单数据变更后，同步到ES中呢？\n\n​    \n\n   这里要考虑的是数据同步的时效性和一致性、对业务代码侵入小、不影响服务本身的性能等。\n\n​    \n\n   **MQ 方案**\n   - ES更新服务作为消费者，接收订单变更MQ消息后对ES进行更新\n\n​    \n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ODNkZWE0Mzc3Mzc1ZWM1MjQzNmFjY2IzZTkzNmMxYjBfVWhqcjRuY2JFUWNrcVlpMkdMQmlWYU9wYUdzUVMwcmhfVG9rZW46Ym94Y25vU213TGFiVDRobEZQbnRaelp1VkxkXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n​    \n\n   **Binlog方案**\n   - ES更新服务借助canal等开源项目，把自己伪装成MySQL的从节点，接收Binlog并解析得到实时的数据变更信息，然后根据这个变更信息去更新ES。\n\n​    \n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=OGZjMzZlZTkxYzk3NmRlYTE2MzJmYmM2ZTAyMmEyZTdfNkxDSjlOeDFob2RYTklJRGp1cmhlbnhhdEZKb21lTkRfVG9rZW46Ym94Y245Rm1xbVVtTGRjWWN0UElXSGcwRGpkXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n​    \n\n   其中BinLog方案比较通用，但实现起来也较为复杂，我们最终选用的是MQ方案。\n\n   因为ES数据只在管理后台使用，对数据可靠性和同步实时性的要求不是特别高。\n\n   考虑到宕机和消息丢失等极端情况，在后台增加了按某些条件手动同步ES数据的功能来进行补偿。\n\n​    \n\n### 3.4 如何安全地更换数据库\n\n   如何将数据从原来的单实例数据库迁移到新的数据库集群，也是一大技术挑战\n\n   不但要确保数据的正确性，还要保证每执行一个步骤后，一旦出现问题，能快速地回滚到上一个步骤。\n\n   我们考虑了停机迁移和不停机迁移的两种方案：\n\n​    \n\n   **（1）不停机迁移方案：**\n\n​    \n\n   - 把旧库的数据复制到新库中，上线一个同步程序，使用 Binlog等方案实时同步旧库数据到新库。\n   - 上线双写订单新旧库服务，只读写旧库。\n   - 开启双写，同时停止同步程序，开启对比补偿程序，确保新库数据和旧库一致。\n   - 逐步将读请求切到新库上。\n   - 读写都切换到新库上，对比补偿程序确保旧库数据和新库一致。\n   - 下线旧库，下线订单双写功能，下线同步程序和对比补偿程序。\n\n​    \n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MjBjZWE5ZTRjMzQ5YTIxNGZkYjg2ODA1NzRiMGMxZDdfUUl5MU0wRWtTdUxrb3JoTnJLZzNvSDM4dXJSOEJrbHZfVG9rZW46Ym94Y25sTGFNMllSb1FmM2dUSTdwbFJLVHBnXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n​    \n\n   **（2）停机迁移方案：**\n\n​    \n\n   - 上线新订单系统，执行迁移程序将两个月之前的订单同步到新库，并对数据进行稽核。\n   - 将商城V1应用停机，确保旧库数据不再变化。\n   - 执行迁移程序，将第一步未迁移的订单同步到新库并进行稽核。\n   - 上线商城V2应用，开始测试验证，如果失败则回退到商城V1应用（新订单系统有双写旧库的开关）。\n\n​    \n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=YjhmZTZiM2RlY2ZjZTIwNTFjODY5NjM0ODcxOTA0MjdfTjZKSWJZWHoyV2M5bk9iUk1DQ2QwT08yWFlaZUhXblRfVG9rZW46Ym94Y242Tmt5NzlDbUo0SG1RZjJpdnFGcWpiXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n​    \n\n   考虑到不停机方案的改造成本较高，而夜间停机方案的业务损失并不大，最终选用的是停机迁移方案。\n\n​    \n\n### 3.5 分布式事务问题\n\n   电商的交易流程中，分布式事务是一个经典问题，比如：\n\n   - 用户支付成功后，需要通知发货系统给用户发货。\n\n   - 用户确认收货后，需要通知积分系统给用户发放购物奖励的积分。\n\n     \n\n   我们是如何保证微服务架构下数据的一致性呢？\n\n   不同业务场景对数据一致性的要求不同，业界的主流方案中，用于解决强一致性的有两阶段提交(2PC)、三阶段提交(3PC)，解决最终一致性的有TCC、本地消息、事务消息和最大努力通知等。\n\n   这里不对上述方案进行详细的描述，介绍一下我们正在使用的本地消息表方案：在本地事务中将要执行的异步操作记录在消息表中，如果执行失败，可以通过定时任务来补偿。\n\n   下图以订单完成后通知积分系统赠送积分为例。\n\n​    \n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MjVkN2Q1NjMyYmRmZmJhODJhNmVmZmVhMGIyNTNkMmRfUDNDZmpnNmFIRllVMzBsODJkd3ZiUmtwV2RKRmxxQTZfVG9rZW46Ym94Y25yNTRtbFZOZGlJcHUwYlNueUdKTHpZXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n​    \n\n### 3.6 系统安全和稳定性\n\n​    \n\n   **网络隔离**\n   - 只有极少数第三方接口可通过外网访问，且都会验证签名，内部系统交互使用内网域名和RPC接口。\n      **并发锁**\n   - 任何订单更新操作之前，会通过数据库行级锁加以限制，防止出现并发更新。\n      **幂等性**\n   - 所有接口均具备幂等性，不用担心对方网络超时重试所造成的影响。\n      **熔断**\n   - 使用Hystrix组件，对外部系统的实时调用添加熔断保护，防止某个系统故障的影响扩大到整个分布式系统中。\n      **监控和告警**\n   - 通过配置日志平台的错误日志报警、调用链的服务分析告警，再加上公司各中间件和基础组件的监控告警功能，让我们能够能够第一时间发现系统异常。\n\n​    \n\n### 3.7  踩过的坑\n\n   **采用MQ消费的方式同步数据库的订单相关数据到ES中，遇到的写入数据不是订单最新数据问题**\n\n​    \n\n   下图左边是原方案：\n\n   在消费订单数据同步的MQ时，如果线程A在先执行，查出数据，这时候订单数据被更新了，线程B开始执行同步操作，查出订单数据后先于线程A一步写入ES中，线程A执行写入时就会将线程B写入的数据覆盖，导致ES中的订单数据不是最新的。\n\n   解决方案是在查询订单数据时加行锁，整个业务执行在事务中，执行完成后再执行下一个线程。\n\n​    \n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ZWUxYjQ1MzZjMjg5ZWE3YWY2YmQwYzBmM2NkNjY1NmRfOEpPSXNwUGhuSWpQNDZLdGlGU0pWMlR1dXhVZXhZaVBfVG9rZW46Ym94Y25MelhveURtTkxsTHNlMWhjbXJqakhnXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n   **sharding-jdbc 分组后排序分页查询出所有数据问题** \n\n​    \n\n   示例：select a  from  temp group by a，b order by a  desc limit 1,10。 \n\n   执行是Sharding-jdbc里group by 和 order by 字段和顺序不一致是将10置为Integer.MAX_VALUE, 导致分页查询失效。\n\n   ```java\n   io.shardingsphere.core.routing.router.sharding.ParsingSQLRouter#processLimit\n   \n   private void processLimit(final List<Object> parameters, final SelectStatement selectStatement, final boolean isSingleRouting) {\n        boolean isNeedFetchAll = (!selectStatement.getGroupByItems().isEmpty() || !selectStatement.getAggregationSelectItems().isEmpty()) && !selectStatement.isSameGroupByAndOrderByItems();\n       selectStatement.getLimit().processParameters(parameters, isNeedFetchAll, databaseType, isSingleRouting);\n   }\n   \n   io.shardingsphere.core.parsing.parser.context.limit.Limit#processParameters\n   \n   /**\n   * Fill parameters for rewrite limit.\n   *\n   * @param parameters parameters\n   * @param isFetchAll is fetch all data or not\n   * @param databaseType database type\n   * @param isSingleRouting is single routing or not\n   */\n   public void processParameters(final List<Object> parameters, final boolean isFetchAll, final DatabaseType databaseType, final boolean isSingleRouting) {\n       fill(parameters);\n       rewrite(parameters, isFetchAll, databaseType, isSingleRouting);\n   }\n   \n   \n   private void rewrite(final List<Object> parameters, final boolean isFetchAll, final DatabaseType databaseType, final boolean isSingleRouting) {\n       int rewriteOffset = 0;\n       int rewriteRowCount;\n       if (isFetchAll) {\n           rewriteRowCount = Integer.MAX_VALUE;\n       } else if (isNeedRewriteRowCount(databaseType) && !isSingleRouting) {\n            rewriteRowCount = null == rowCount ? -1 : getOffsetValue() + rowCount.getValue();\n       } else {\n          rewriteRowCount = rowCount.getValue();\n       }\n       if (null != offset && offset.getIndex() > -1 && !isSingleRouting) {\n          parameters.set(offset.getIndex(), rewriteOffset);\n        }\n        if (null != rowCount && rowCount.getIndex() > -1) {\n           parameters.set(rowCount.getIndex(), rewriteRowCount);\n         }\n   }\n   \n   ```\n\n​    \n\n   正确的写法应该是  select a  from  temp group by a desc ，b limit 1,10 ；  使用的版本是sharing-jdbc的3.1.1。\n\n   ES分页查询如果排序字段存在重复的值，最好加一个唯一的字段作为第二排序条件，避免分页查询时漏掉数据、查出重复数据，比如用的是订单创建时间作为唯一排序条件，同一时间如果存在很多数据，就会导致查询的订单存在遗漏或重复，需要增加一个唯一值作为第二排序条件或者直接使用唯一值作为排序条件。\n\n\n\n## 四、成果\n\n   - 一次性上线成功，稳定运行了一年多\n\n   - 核心服务性能提升十倍以上\n\n   - 系统解耦，迭代效率大幅提升\n\n   - 能够支撑商城至少五年的高速发展\n\n     \n\n## 五、结语\n\n   我们在系统设计时并没有一味追求前沿技术和思想，面对问题时也不是直接采用主流电商的解决方案，而是根据业务实际状况来选取最合适的办法。\n\n   个人觉得，一个好的系统不是在一开始就被大牛设计出来的，一定是随着业务的发展和演进逐渐被迭代出来的，持续预判业务发展方向，提前制定架构演进方案，简单来说就是：走到业务的前面去！","date":"2020-12-23","author":"SphereEx","excerpt":null,"metaDescription":null,"createdAt":"2023-03-30T07:13:27.915Z","updatedAt":"2023-03-30T07:13:28.692Z","publishedAt":"2023-03-30T07:13:28.687Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":46,"attributes":{"feature":false,"title":"分布式数据库解决方案——核心优势","content":"#分布式数据库解决方案——核心优势\n\n随着数字化时代的到来数据的规模和复杂性不断增加。传统的单机数据库面临性能和可靠性受到限制的挑战。为了满足当今大规模数据处理的需求[分布式数据库解决方案](linkhttps://www.sphere-ex.cn/solution?tab=1/)将数据存储和处理任务分散在多台服务器上，允许系统横向扩展并提供了更好的性能和可用性。\n![output.png](https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/output_cb490a248b.png)\n\n可扩展性\n\n分布式的数据库解决方案的主要优势是可扩展性。分布式的数据库允许将数据划分成更小的部分，分布在不同的服务器上。这使得系统能够水平扩展通过添加更多的服务器来增加整体处理能力。当业务需要处理更多的数据时只需简单地增加节点而不需要改变数据库架构。这种可扩展性使得分布式的数据库适合应对日益增长的数据负载。\n\n高可用性\n\n分布式数据库解决方案还提供了高可用性。在传统单机数据库中如果服务器发生故障或停机整个系统可能会崩溃导致数据不可访问。而在分布式数据库解决方案中数据被复制到多个节点上，当一个节点发生故障时其他节点仍然可以继续提供服务。这种冗余性可以大大减少系统宕机的风险保障数据的持续可用性。\n\n容错性\n\n分布式的数据库解决方案的容错性是指系统在面对故障或异常情况时能够继续运行而不崩溃。分布式的数据库通过数据冗余和复制来提高容错性。当一个节点发生故障时系统可以从其他节点恢复数据保持服务的连续性。容错性使得分布式的数据库能够应对硬件故障、网络问题和其他意外事件并提供更可靠的数据存储和处理解决方案。\n\n分布式数据库解决方案还具有较高的灵活性。由于数据被分散存储在多个节点上系统可以根据需要动态调整和重新配置。这种灵活性使得在业务需求变化时能够快速响应并支持各种数据处理模式。可以根据数据的访问模式将热数据存储在性能较高的节点上，而将冷数据存储在成本较低的节点上。灵活的配置可以优化资源利用并提供更高效的数据管理。","date":null,"author":null,"excerpt":"随着数字化时代的到来数据的规模和复杂性不断增加。传统的单机数据库面临性能和可靠性受到限制的挑战。为了满足当今大规模数据处理的需求分布式数据库解决方案将数据存储和处理任务分散在多台服务器上，允许系统横向扩展并提供了更好的性能和可用性。","metaDescription":null,"createdAt":"2023-08-16T01:07:49.417Z","updatedAt":"2023-08-16T01:09:29.848Z","publishedAt":"2023-08-16T01:07:57.905Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":{"id":496,"attributes":{"name":"output.png","alternativeText":"output.png","caption":"output.png","width":1460,"height":606,"formats":{"thumbnail":{"name":"thumbnail_output.png","hash":"thumbnail_output_cb490a248b","ext":".png","mime":"image/png","width":245,"height":102,"size":11.33,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/thumbnail_output_cb490a248b.png"},"large":{"name":"large_output.png","hash":"large_output_cb490a248b","ext":".png","mime":"image/png","width":1000,"height":415,"size":76.33,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/large_output_cb490a248b.png"},"medium":{"name":"medium_output.png","hash":"medium_output_cb490a248b","ext":".png","mime":"image/png","width":750,"height":311,"size":50.96,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/medium_output_cb490a248b.png"},"small":{"name":"small_output.png","hash":"small_output_cb490a248b","ext":".png","mime":"image/png","width":500,"height":208,"size":29.25,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/small_output_cb490a248b.png"}},"hash":"output_cb490a248b","ext":".png","mime":"image/png","size":84.52,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/output_cb490a248b.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2023-08-16T01:02:06.595Z","updatedAt":"2023-08-16T01:02:06.595Z"}}},"metaLogo":{"data":{"id":496,"attributes":{"name":"output.png","alternativeText":"output.png","caption":"output.png","width":1460,"height":606,"formats":{"thumbnail":{"name":"thumbnail_output.png","hash":"thumbnail_output_cb490a248b","ext":".png","mime":"image/png","width":245,"height":102,"size":11.33,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/thumbnail_output_cb490a248b.png"},"large":{"name":"large_output.png","hash":"large_output_cb490a248b","ext":".png","mime":"image/png","width":1000,"height":415,"size":76.33,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/large_output_cb490a248b.png"},"medium":{"name":"medium_output.png","hash":"medium_output_cb490a248b","ext":".png","mime":"image/png","width":750,"height":311,"size":50.96,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/medium_output_cb490a248b.png"},"small":{"name":"small_output.png","hash":"small_output_cb490a248b","ext":".png","mime":"image/png","width":500,"height":208,"size":29.25,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/small_output_cb490a248b.png"}},"hash":"output_cb490a248b","ext":".png","mime":"image/png","size":84.52,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/output_cb490a248b.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2023-08-16T01:02:06.595Z","updatedAt":"2023-08-16T01:02:06.595Z"}}},"localizations":{"data":[]}}},{"id":47,"attributes":{"feature":false,"title":"分布式数据库服务商","content":"# 分布式数据库服务商\n\n分布式数据库是将数据库系统部署在多台计算机上，通过网络连接进行数据的存储和访问，以实现数据的共享和处理。分布式的数据库旨在解决传统单机数据库在可靠性和扩展性等方面的局限性。\n\n![1280X1280.PNG](https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/1280_X1280_b8d9276b9d.PNG)\n\n架构与设计\n\n分布式的数据库服务商会提供架构和设计方面的服务以满足不同应用场景的需求。服务商会考虑客户的数据量、访问模式、数据一致和故障容忍能力等因素来设计合适的分布式的数据库架构。这可能涉及到主从复制、分区划分、数据分片、数据备份与恢复策略等。\n\n性能优化与调优\n\n性能是一个重要的关注点，特别是对于大规模分布式的数据库系统而言。分布式数据库服务商会提供性能优化与调优服务，通过对数据库的索引优化、查询优化和数据存储格式优化等手段，提高数据库的响应速度和处理能力以确保高效的数据访问。\n\n数据安全与隐私保护\n\n数据安全是任何数据库系统的重要组成部分。对于分布式的数据库来说数据的存储和传输涉及到更多的风险。分布式的数据库服务商会提供数据加密、权限管理、访问控制等安全措施以保护客户数据的安全和隐私并确保敏感信息不被未授权的人员获取。\n\n故障恢复与容灾备份\n\n由于分布式的数据库系统涉及多台计算机，故障恢复和容灾备份变得尤为重要。分布式数据库服务商会提供故障检测、故障定位、数据备份与恢复等服务以确保在发生故障时数据库系统能够尽快恢复并避免数据丢失和业务中断。\n\n随着业务的发展数据库的需求可能会不断增加。分布式数据库服务商会提供扩展性与弹性伸缩的服务以应对数据量和访问压力的增加。服务商可能会采用水平扩展或垂直扩展的方式来满足客户的需求确保数据库系统能够随着业务的增长而持续稳定运行。","date":null,"author":null,"excerpt":"分布式数据库是将数据库系统部署在多台计算机上，通过网络连接进行数据的存储和访问，以实现数据的共享和处理。分布式的数据库旨在解决传统单机数据库在可靠性和扩展性等方面的局限性。","metaDescription":null,"createdAt":"2023-08-16T01:13:55.291Z","updatedAt":"2023-08-16T01:13:56.986Z","publishedAt":"2023-08-16T01:13:56.983Z","locale":"zh","caseStudiesType":{"data":{"id":4,"attributes":{"name":"电商","createdAt":"2021-12-11T23:02:39.013Z","updatedAt":"2021-12-11T23:02:41.225Z","publishedAt":"2021-12-11T23:02:41.156Z","locale":"zh"}}},"cover":{"data":{"id":497,"attributes":{"name":"1280X1280.PNG","alternativeText":"1280X1280.PNG","caption":"1280X1280.PNG","width":1280,"height":687,"formats":{"thumbnail":{"name":"thumbnail_1280X1280.PNG","hash":"thumbnail_1280_X1280_b8d9276b9d","ext":".PNG","mime":"image/png","width":245,"height":132,"size":26.03,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/thumbnail_1280_X1280_b8d9276b9d.PNG"},"large":{"name":"large_1280X1280.PNG","hash":"large_1280_X1280_b8d9276b9d","ext":".PNG","mime":"image/png","width":1000,"height":537,"size":293.26,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/large_1280_X1280_b8d9276b9d.PNG"},"medium":{"name":"medium_1280X1280.PNG","hash":"medium_1280_X1280_b8d9276b9d","ext":".PNG","mime":"image/png","width":750,"height":403,"size":190.87,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/medium_1280_X1280_b8d9276b9d.PNG"},"small":{"name":"small_1280X1280.PNG","hash":"small_1280_X1280_b8d9276b9d","ext":".PNG","mime":"image/png","width":500,"height":268,"size":83.52,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/small_1280_X1280_b8d9276b9d.PNG"}},"hash":"1280_X1280_b8d9276b9d","ext":".PNG","mime":"image/png","size":357,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/1280_X1280_b8d9276b9d.PNG","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2023-08-16T01:12:51.296Z","updatedAt":"2023-08-16T01:12:51.296Z"}}},"metaLogo":{"data":{"id":497,"attributes":{"name":"1280X1280.PNG","alternativeText":"1280X1280.PNG","caption":"1280X1280.PNG","width":1280,"height":687,"formats":{"thumbnail":{"name":"thumbnail_1280X1280.PNG","hash":"thumbnail_1280_X1280_b8d9276b9d","ext":".PNG","mime":"image/png","width":245,"height":132,"size":26.03,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/thumbnail_1280_X1280_b8d9276b9d.PNG"},"large":{"name":"large_1280X1280.PNG","hash":"large_1280_X1280_b8d9276b9d","ext":".PNG","mime":"image/png","width":1000,"height":537,"size":293.26,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/large_1280_X1280_b8d9276b9d.PNG"},"medium":{"name":"medium_1280X1280.PNG","hash":"medium_1280_X1280_b8d9276b9d","ext":".PNG","mime":"image/png","width":750,"height":403,"size":190.87,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/medium_1280_X1280_b8d9276b9d.PNG"},"small":{"name":"small_1280X1280.PNG","hash":"small_1280_X1280_b8d9276b9d","ext":".PNG","mime":"image/png","width":500,"height":268,"size":83.52,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/small_1280_X1280_b8d9276b9d.PNG"}},"hash":"1280_X1280_b8d9276b9d","ext":".PNG","mime":"image/png","size":357,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/1280_X1280_b8d9276b9d.PNG","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2023-08-16T01:12:51.296Z","updatedAt":"2023-08-16T01:12:51.296Z"}}},"localizations":{"data":[]}}}],"meta":{"pagination":{"page":1,"pageSize":25,"pageCount":2,"total":26}}},"en":{"data":[{"id":15,"attributes":{"feature":true,"title":"Apache ShardingSphere & Energy Monster","content":"[Energy Monster](https://ir.enmonster.com/)’s application of [ShardingSphere-JDBC](https://shardingsphere.apache.org/document/current/en/overview/#shardingsphere-jdbc)\n\n[Energy Monster](https://ir.enmonster.com/) is a consumer tech company with the mission to energize everyday life. The company is the largest provider of mobile device charging services in Asia.\n\nAs the company’s business concurrency volume is getting larger, the amount of data generated (users, orders, activities, etc.) increases each day. The traditional relational database has proven to be inadequate in supporting millions or tens of millions of data volumes in a single database or table.\n\nPerformance has been unable to meet the benchmark requirements of business development. Under these circumstances, data sharding is an effective way to solve the problem.\n\n## Technology selection\nUnder the [Database Plus](https://shardingsphere.apache.org/) concept, ShardingSphere is designed to build an ecosystem on top of heterogeneous databases. The goal is to provide globally scalable and enhanced computing capabilities while maximizing the original database computing capabilities.\n\nThe interaction between applications and databases becomes oriented towards the Database Plus standard, therefore minimizing the impact of database fragmentation on upper-layer services.\n\nWithin the ShardingSphere ecosystem, [ShardingSphere-JDBC](https://shardingsphere.apache.org/document/current/en/overview/#shardingsphere-jdbc) is positioned as a lightweight Java framework, providing additional services in Java’s JDBC layer.\n\nIt uses the client to directly connect to the database and provide services in the form of a jar package, without additional deployment and dependence. It can be understood as an enhanced version of the JDBC driver, which is fully compatible with JDBC and various ORM frameworks.\n\nShardingSphere-JDBC enables developers to focus only on the work outside the data layer by coordinating the data read and write under the data sharding, instead of using business code to manually select databases and tables.\n\n## Business case\nUCS is Energy Monster’s user-centric service providing basic functionality for users on the Server side. In 2018, it was stripped from [PHP](https://www.php.net/) Server and moved to the Java technology stack to implement microservitization.\n\nIt involves the design of new databases and tables and data cleaning and migration. The whole switchover process was expected to ensure the following functions:\n\n- Stability: smooth release in a short time without halting.\n\n- Accuracy: ensure accurate cleaning of tens of millions of data volumes.\n\n- Scalability: solve the performance problems caused by increasing data volume and ensure scalability.\n\n> Solutions to data cleansing and migration\n\n- Initial data synchronization.\n- The application’s server cuts off the entry (users).\n- Data synchronization (updates and new users since the last time point).\n- Data cleaning.\n- User center release.\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/2t8jnqgh5pfimhxa07id.png)\n\n\n> Data sharding strategy\n\nThe database adopts a database shards design, divided into 16 databases. The default shard key is `user_id` and the default sharding strategy `user_id` is mod 16, such as `${user_id % 16}` for the user table. For SQL that does not carry shard keys, broadcast routing is used.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/t6nihbk4du8d9h1sd5sf.png)\n\n\n`user_id` is used as the shard key because `user_id` can cover most business scenarios, and other fields possibly can be empty. In the local test, the query of shard key strategy (openId,mobile) took 50ms to 200ms.\n\n> Using the sharding algorithm\n\nThere are currently three sharding algorithms available.\n\n- Standard sharding algorithm. It corresponds to `StandardShardingAlgorithm`, used for scenarios that use a single key as the shard key, such as =, IN, BETWEEN AND, >, <, > =, < =.\n- Complex sharding algorithm. It corresponds to `ComplexKeysShardingAlgorithm`, used for scenarios that use multi-key as the shard key. The logic with multiple shard keys is complex and requires developers to handle it by themselves.\n- Hint sharding algorithm. It corresponds to `HintShardingAlgorithm`, used for scenarios where the Hint row is used for sharding.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/5uv7u0nrglequ83gn4ky.png)\n\n\n## Upgrading ShardingSphere-JDBC\n\nShardingSphere-JDBC is used in multiple business scenarios, such as order, inventory, and finance. By 2021, the R&D groups or teams were using different versions of ShardingSphere-JDBC, ranging from 1.X to 4.X, which is difficult to achieve unified maintenance in the later stage.\n\nAdditionally, there are some potential bugs and missing functions in the earlier version. Based on requirements for unified management and availability, we implemented a project to unify the ShardingSphere-JDBC’s versions used by the company and upgrade them to a 4.1.1 stable version in April 2021.\n\n**The following problems were encountered during the upgrade:**\n\n**1. It takes a long time to start the service after the upgrade.**\n\nShardingSphere-JDBC checks the metadata consistency of sub-tables when the service is started. The configuration item `max.connections.size.per.quer` (maximum number of connections that can be opened per query) is 1 by default. With a large number of tables, the loading process would be slow. You need to refer to the connection pool configuration to improve the loading speed.\n\n**2. There is no response when there is no shard key in the sub-table query.**\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/641dysf5zoizgr8bmjuq.png)\n\n\nLogical SQL query does not specify shard keys and it queries all the tables according to the whole database tables router in broadcasting routing.\n\nThe configuration items have 108 pieces of real tables in a database. According to the configuration of `maxConnectionsizeperquery=50`, ShardingSphere-JDBC uses the connection limit mode, divides the query requests into three groups, and merges the results with in-memory. As a result, 36 database connections are required for one query. But the `maxActive` configured by the [druid](https://druid.apache.org/) thread pool is set to 20, resulting in a deadlock.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/ydv03pncno3wsd23y7r1.png)\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/61blt38kcoukha8aguob.png)\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/2c2w85mdryc53gh4ope5.png)\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/568lubr5zg52ropjw374.png)\n\n\n\n**Solutions:**\n\n- Combine `check.table.metadata.enabled=true`（check the metadata consistency in sub-tables when started）and properly configure - `maxConnectionSizePerQuery`（maximum number of connections that can be opened by each query).\n- `maxConnectionSizePerQuery` should be less than the maximum number of active threads configured by the druid thread pool.\n\n**3. After upgrading from 1.X, an error message “Cannot update Sharding key” is displayed in SQL execution, and the actual shard key value is not updated.**\n\nTo avoid data query failure caused by changing the shard key value, shard key detection is added to the `SQL update` in the 4.X version. The error can be rectified in the following ways:\n\n- remove the shard key when updating.\n- the shard key is added to the `where` statement synchronously.\n\n**4. A start failure is caused when using `druid-spring-boot-starter`, which is incompatible with `Sharding-datasource`.**\n\nThe druid data connection pool starter will load and create a default data source. This will cause conflicts when ShardingSphere-JDBC creates data sources.\n\n**5. `inline strategy` reports an error in range query.**\n\nThe `inline strategy` doesn't support range query by default and the `standard strategy` is advised. Add the following configuration if the `inline strategy` is needed for the range query.\n\n`spring.shardingsphere.props.allow.range.query.with.inline.sharding: true`\n\n**Note:** Here all the `inline strategy` range queries will query each sub-table in broadcasting.\n\n**6. The “Cannot find owner from table” error is reported.**\n\nSQL (simplified):\n\n`select id from` (select id from x) as a group by a.id\n\nThe 4.X version supports limited sub-queries. This problem is caused by the name of the intermediate table. Remove the table alias of `select` or `group order` or other fields.\n\n> [https://github.com/apache/shardingsphere/issues/4810](https://github.com/apache/shardingsphere/issues/4810)\n\n**7. The table’s primary key conflicts with the primary key generated by the [SNOWFLAKE](https://programming.vip/docs/overview-of-snowflake-algorithm.html) algorithm.**\n\nShardingSphere provides flexible ways to configure distributed primary key generation strategies. In the sharding rule configuration module, you can configure the primary key generation strategy for each table.\n\nBy default, the [snowflake](https://programming.vip/docs/overview-of-snowflake-algorithm.html) algorithm is used to generate long integer data of 64bit. The snowflake generator needs to be configured with:\n\n`spring.shardingsphere.sharding.tables.x.key-generator.props.worker.id = ${dcc.node.id}`\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/dbopmsazvtztkb0p2m0z.png)\n\n\n\nThe company uses the [apollo](https://www.apolloconfig.com/#/) configuration center to deliver the node id of the service instance. The service uses multi-data sources. If you use the YAML file to load sharding configuration, the `workId` cannot be automatically loaded into sharding configuration items.\n\n**Solutions:**\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/mfxy2xgsl6w0bwgi0hqv.png)\n\n\nUse the custom generator type based on the built-in `SnowflakeShardingKeyGenerator`.\n\nIf the primary key is used as a shard key, configure `max.vibration.offset` based on the data sharding value to increase the vibration range.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/7jo25gkyz57kyttllvsd.png)\n\n\n\n**8. The 3.X version reports an error when `CASE WHEN` statement is executed.**\n\nFirst, the 3.X and 4.X versions don’t support the `case when` statement.\n\nThe 3.X and 4.X versions have different logics when parsing the shard keys of `case when`'s `update` statement. The 4.X `parserEngine.parse` method will ignore the `case when` parsing parameters, resulting in inconsistency with the external parameter list and an error when 3.X executes the normal SQL.\n\nThe 3.X version works correctly because the first parameter of `case when` is intentionally set to the shard key when the SQL is written, and the `case when` statement comes first.\n\n> [https://github.com/apache/shardingsphere/issues/13233](https://github.com/apache/shardingsphere/issues/13233)\n\n**Solutions：**\n\n- It is suggested to rewrite SQL as the `case when` is not supported.\n- According to the shard key parsing logic in version 4.1.1, `case when` is placed at the end, and the shard key remains the first parameter of `case when`.\n\n**9. The logical table `actualDataNodes` is configured and no default value error is reported for the primary key.**\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/7tpr0fitx420itauw5f8.png)\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/pprbsvn5n8tdrm2mezs7.png)\n\n\n\n\nThe `check.table.metadata.enabled=true` is not configured for service, and the metadata consistency of sub-tables is not checked by default.\n\nThe first table of `actualDataNodes` configured by services does not exist, resulting in an empty `GenerateKeyContenxt`.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/a75kpppwhniaubgw3iti.png)\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/cw16ejpxmwvcva9bd9t4.png)\n\n\n\n\n**Solutions：**\n\n- Configure `check.table.metadata.enabled=true`. A non-existent table is detected when started and an error is reported.\n- Rewrite the `actualDataNodes inline` expression to make sure that the first table exists.\n\n**10. In version 3.0, there is a deadlock under the high concurrency of the full database and table router.**\n\nShardingSphere-JDBC uses local transactions by default. In local transactions, the database connection is obtained asynchronously. Under high concurrency, it is possible that all database connections cannot be obtained, resulting in a deadlock.\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/5eo47dqy991nmvu048k8.png)\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/hdnnfskjri9un1wj6n3h.png)\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/y8fpd887zvkg5oovt4hm.png)\n\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/pzqw4c89j1mplfmwfxz0.png)\n\n\n\n\n## Conclusion\n\nAs a [ShardingSphere](https://shardingsphere.apache.org/) core user, [Energy Monster](https://ir.enmonster.com/)’s upgrade process also reflects some problems that community users may encounter in the application of ShardingSphere.\n\nCurrently, Apache ShardingSphere’s stable version has been updated to 5.1.2 and has been optimized in terms of its functions, performance, testing, documentation, and examples.\n\nYou can refer to [Apache ShardingSphere’s official website](https://shardingsphere.apache.org/) for more information. If you have any questions or suggestions, you are also welcome to give feedback on [Github](https://github.com/apache/shardingsphere). The community will actively respond and discuss.\n\n**Project Links:**\n\n[ShardingSphere Github](https://github.com/apache/shardingsphere/issues?page=1&q=is%3Aopen+is%3Aissue+label%3A%22project%3A+OpenForce+2022%22)\n\n[ShardingSphere Twitter](https://twitter.com/ShardingSphere)\n\n[ShardingSphere Slack](https://join.slack.com/t/apacheshardingsphere/shared_invite/zt-sbdde7ie-SjDqo9~I4rYcR18bq0SYTg)\n\n[Contributor Guide](https://shardingsphere.apache.org/community/cn/contribute/)\n\n[GitHub Issues\n](https://github.com/apache/shardingsphere/issues)\n[Contributor Guide](https://shardingsphere.apache.org/community/en/contribute/)","date":"2022-09-06","author":"SphereEx","excerpt":"NASDAQ-listed Energy Monster is the largest provider of mobile device charging services in Asia. This article illustrates how Energy Monster uses ShardingSphere-JDBC.","metaDescription":"Energy Monster is a power bank rental company, allowing its users to rent power banks by scanning QR codes. It is designed to help people to live their life without interruptions. Energy Monster reportedly covers 100 cities and 100K merchants. ","createdAt":"2022-09-26T06:27:13.410Z","updatedAt":"2022-09-28T02:42:03.843Z","publishedAt":"2022-09-27T06:55:09.446Z","locale":"en","caseStudiesType":{"data":{"id":9,"attributes":{"name":"Internet","createdAt":"2022-09-26T10:34:03.086Z","updatedAt":"2022-09-27T06:59:34.838Z","publishedAt":"2022-09-27T06:59:34.836Z","locale":"en"}}},"cover":{"data":{"id":334,"attributes":{"name":"energy monster.png","alternativeText":"energy monster.png","caption":"energy monster.png","width":1093,"height":489,"formats":{"thumbnail":{"name":"thumbnail_energy monster.png","hash":"thumbnail_energy_monster_2f744c0ab8","ext":".png","mime":"image/png","width":245,"height":110,"size":18,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_energy_monster_2f744c0ab8.png"},"large":{"name":"large_energy monster.png","hash":"large_energy_monster_2f744c0ab8","ext":".png","mime":"image/png","width":1000,"height":447,"size":191.25,"path":null,"url":"https://official-media.sphere-ex.com/large_energy_monster_2f744c0ab8.png"},"medium":{"name":"medium_energy monster.png","hash":"medium_energy_monster_2f744c0ab8","ext":".png","mime":"image/png","width":750,"height":336,"size":115.92,"path":null,"url":"https://official-media.sphere-ex.com/medium_energy_monster_2f744c0ab8.png"},"small":{"name":"small_energy monster.png","hash":"small_energy_monster_2f744c0ab8","ext":".png","mime":"image/png","width":500,"height":224,"size":58.44,"path":null,"url":"https://official-media.sphere-ex.com/small_energy_monster_2f744c0ab8.png"}},"hash":"energy_monster_2f744c0ab8","ext":".png","mime":"image/png","size":182.88,"url":"https://official-media.sphere-ex.com/energy_monster_2f744c0ab8.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T06:34:45.604Z","updatedAt":"2022-09-26T06:34:45.604Z"}}},"metaLogo":{"data":{"id":349,"attributes":{"name":"Energy monster logo.png","alternativeText":"Energy monster logo.png","caption":"Energy monster logo.png","width":373,"height":298,"formats":{"thumbnail":{"name":"thumbnail_Energy monster logo.png","hash":"thumbnail_Energy_monster_logo_102b9745bc","ext":".png","mime":"image/png","width":195,"height":156,"size":8.8,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_Energy_monster_logo_102b9745bc.png"}},"hash":"Energy_monster_logo_102b9745bc","ext":".png","mime":"image/png","size":17.3,"url":"https://official-media.sphere-ex.com/Energy_monster_logo_102b9745bc.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:19:26.166Z","updatedAt":"2022-09-26T08:19:26.166Z"}}},"localizations":{"data":[]}}},{"id":16,"attributes":{"feature":false,"title":"Apache ShardingSphere & China Unicom Digital Technology: Ensuring Stability, Efficiency, and Replicability at Scale","content":"China Unicom Digital Tech is the subsidiary of [China Unicom](https://www.chinaunicom.com.hk/en/global/home.php), a global leading telecommunication operator with over 310 million subscribers as of 2021.\n\nThe company integrates with China Unicom’s capabilities such as cloud computing, Big Data, IoT, AI, blockchain, and security and touches on smart city, industrial internet, ecological protection, cultural tourism, and other fields.\n\nUnicom Digital Tech has accumulated a large number of industry benchmark cases and successfully provided customers with diverse and professional products & services.\n\n## Background\nIn recent years, dozens of service hotline platforms have been launched with the help of Unicom Digital Tech, in a bid to improve enterprise and government services.\n\nThe service hotlines are characterized by high concurrency and large amounts of data. Every time we dial a hotline, a work order record is generated. The business volumes of a hotline during the epidemic have increased several times compared to the past.\n\nIn the work order module of the government or emergency services hotlines, to meet the business needs of massive amounts of data and high stability, Unicom Digital Tech adopts [ShardingSphere](https://shardingsphere.apache.org/) to carry out sharding and store work order information.\n\n## Business challenges\nGovernment service hotlines are the main channel through which the government interacts with enterprises and the public. It provides 24/7 services for the public through a single telephone number.\n\nIn addition to dealing with work orders, it also involves services such as telephone traffic, Wiki, voice chat, etc.\n\nIn the system planning stage, the database layer is the cornerstone of the business system, so the technology selection process is crucial. System stability is a hard indicator, followed by performance, which directly affects the hotline’s service capability.\n\nAt the same time, it should allow for easy maintenance and management, and facilitate system upgrades and backend maintenance.\n\nThe hotline service raises the following requirements for database architecture selection:\n\n- Maturity and stability\n- High performance\n- Easy maintenance\n- Low coupling of business code\n\n## Why did Unicom Digital Technology choose ShardingSphere?\nThe technical team conducted several rounds of research and tests in terms of stability, features, access mode, and product performance, and they finally choose the “ShardingSphere + [MySQL](https://www.mysql.com/)” distributed database solution.\n\n- **Advanced concept**\n\nAn increasing number of application scenarios have exacerbated database fragmentation. [Database Plus](https://faun.pub/whats-the-database-plus-concepand-what-challenges-can-it-solve-715920ba65aa) is designed to build an ecosystem on top of fragmented heterogeneous databases and provide enhancements capability for the database.\n\nAdditionally, it can prevent database binding, achieve independent control, add more features to the original database, and improve the overall feature ceiling of the data infrastructure.\n\n- **Maturity and stability**\n\nThe ShardingSphere project started in 2016. Following years of R&D and iterations, the project has been polished and has proven its stability, maturity, and reliability in multiple Internet scenarios.\n\n- **Comprehensive features**\n\nIn addition to sharding capability, ShardingSphere is also capable of data encryption & decryption and shadow DB, which were also among the evaluation indicators of the technical team.\n\nThe scenario requirements of security and stress testing can be met by a set of technology stacks. ShardingSphere’s capabilities provide improved support for the construction and architecture expansion of the hotline system.\n\n- **High compatibility / easy access**\n\nShardingSphere is compatible with the MySQL protocol and supports multiple syntaxes. In the development process, there is almost no need to worry about access to SQL as it is really convenient.\n\n- **High performance**\n\nWhen the hotline is busy, thousands of on-duty operators are online. The work order record includes tens of millions of records, which create a certain demand for performance.\n\nShardingSphere-JDBC is positioned as a lightweight Java framework with ideal stress test results and data, which can meet the requirements of government system service capabilities.\n\n- **Simplified operation & maintenance**\n\nOn the basis of JDBC’s high performance, ShardingSphere also provides a Proxy friendly to ops teams, which can be directly accessed using common clients.\n\nIn addition to the trade-offs of the above five key considerations, other database middleware products were also taken into consideration. The following table indicates the comparison between ShardingSphere and [MyCat](http://mycat.sourceforge.net/).\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/mc2d1n3csc1bf9vu26ab.png)\n \n\n## Solutions\nAs “ShardingSphere+MySQL” ensures stability, ease of use, and ultimate performance, this configuration has been replicated and used in many government and enterprise service hotline projects of Unicom Digital Tech.\n\n- **Deployment mode**\nThe combination of [ShardingSphere-JDBC](https://shardingsphere.apache.org/document/current/en/overview/#shardingsphere-jdbc) and [ShardingSphere-Proxy](https://shardingsphere.apache.org/document/current/en/quick-start/shardingsphere-proxy-quick-start/) ensures both system performance and maintainability.\n\n- **Sharding key / sharding algorithm**\nThe work order table of the hotline service is relatively large, and the logic is simple to use. Therefore, the work order ID is selected as the sharding key, and the data is distributed through the hash algorithm.\n\n- **Migration**\nThe new project does not involve historical data, so there is no need to consider the data migration process, and it can be directly used when it goes live.\n\n- **Implementation process**\nOwing to the comprehensive evaluation of the technical scheme and multiple copies of the scheme, the implementation process is smooth.\n\n- **System architecture**\nThe hotline service module adopts the micro-service architecture\n\nThe business module of the hotline service adopts the micro-service architecture, with nodes ranging from ten to dozens, depending on the scale of specific provinces and cities. At the database layer, physical machines are used to deploy three-node MHA. Each hotline system requires four sets of MHA in general.\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/v8k9lv1l3pjxme0e7sxs.png)\n \n\n## Advantages\n\n> Business\n\n-  the client mode. It ensures the government hotline’s requirements on performance to the greatest degree.\n- System stability provides strong support for business continuity.\n\n> R&D\n\n- Excellent compatibility. It is compatible with mainstream database SQL syntax and can be used after docking.\n- Comprehensive features. It supports read/write splitting, data encryption, and full-link stress testing, with strong system scalability.\n\n> OPS\n\n- It supports the proxy mode and provides easy maintenance for Ops teams.\n\n> Technology selection\n\n- It prevents database binding and provides enough flexibility for future upgrades.\n\n## Conclusion\nThe hotline service cases of China Unicom Digital Tech verified ShardingSphere’s capability to support government service scenarios and further proved that ShardingSphere can be used in any industry.\n\n## Project Links:\n[ShardingSphere Github](https://github.com/apache/shardingsphere/issues?page=1&q=is%3Aopen+is%3Aissue+label%3A%22project%3A+OpenForce+2022%22)\n\n[ShardingSphere Twitter](https://github.com/apache/shardingsphere/issues?page=1&q=is%3Aopen+is%3Aissue+label%3A%22project%3A+OpenForce+2022%22)\n\n[ShardingSphere Slack](https://join.slack.com/t/apacheshardingsphere/shared_invite/zt-sbdde7ie-SjDqo9~I4rYcR18bq0SYTg)\n\n[Contributor Guide](https://shardingsphere.apache.org/community/cn/contribute/)","date":"2022-07-20","author":"SphereEx","excerpt":"China Unicom Digital Technology, a global telco operator with 310 million subscribers, chose ShardingSphere to manage its gov and enterprise hotlines data infrastructure.","metaDescription":"China Unicom is a company that provides broadband communications and information services. It has subsidiaries in 31 provinces (autonomous regions and municipalities) across China and many countries and regions around the world.","createdAt":"2022-09-26T06:38:43.311Z","updatedAt":"2022-09-27T06:55:16.269Z","publishedAt":"2022-09-27T06:55:16.266Z","locale":"en","caseStudiesType":{"data":{"id":7,"attributes":{"name":"Telecom","createdAt":"2022-09-26T10:29:57.244Z","updatedAt":"2022-09-27T06:59:48.158Z","publishedAt":"2022-09-27T06:59:48.156Z","locale":"en"}}},"cover":{"data":{"id":335,"attributes":{"name":"China unicom.png","alternativeText":"China unicom.png","caption":"China unicom.png","width":722,"height":615,"formats":{"thumbnail":{"name":"thumbnail_China unicom.png","hash":"thumbnail_China_unicom_483a43110d","ext":".png","mime":"image/png","width":183,"height":156,"size":11.99,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_China_unicom_483a43110d.png"},"small":{"name":"small_China unicom.png","hash":"small_China_unicom_483a43110d","ext":".png","mime":"image/png","width":500,"height":426,"size":46.22,"path":null,"url":"https://official-media.sphere-ex.com/small_China_unicom_483a43110d.png"}},"hash":"China_unicom_483a43110d","ext":".png","mime":"image/png","size":61.21,"url":"https://official-media.sphere-ex.com/China_unicom_483a43110d.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T06:39:16.946Z","updatedAt":"2022-09-26T06:39:16.946Z"}}},"metaLogo":{"data":{"id":350,"attributes":{"name":"China unicom logo.png","alternativeText":"China unicom logo.png","caption":"China unicom logo.png","width":750,"height":571,"formats":{"thumbnail":{"name":"thumbnail_China unicom logo.png","hash":"thumbnail_China_unicom_logo_2aba6471b9","ext":".png","mime":"image/png","width":205,"height":156,"size":13.18,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_China_unicom_logo_2aba6471b9.png"},"small":{"name":"small_China unicom logo.png","hash":"small_China_unicom_logo_2aba6471b9","ext":".png","mime":"image/png","width":500,"height":381,"size":31.14,"path":null,"url":"https://official-media.sphere-ex.com/small_China_unicom_logo_2aba6471b9.png"}},"hash":"China_unicom_logo_2aba6471b9","ext":".png","mime":"image/png","size":15.23,"url":"https://official-media.sphere-ex.com/China_unicom_logo_2aba6471b9.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:28:31.851Z","updatedAt":"2022-09-26T08:28:31.851Z"}}},"localizations":{"data":[]}}},{"id":17,"attributes":{"feature":false,"title":"Heterogeneous migration: reducing Dangdang’s customer system RTO 60x and increasing speed by 20%","content":"> [Apache ShardingSphere](https://shardingsphere.apache.org/) helps [Dangdang](https://www.crunchbase.com/organization/dangdang-com) rebuild its customer system with 350 million users, and seamlessly transition from a [PHP](https://www.php.net/)+[SQL Server](https://www.microsoft.com/en-us/sql-server/sql-server-downloads) technology stack to a Java+ShardingSphere+[MySQL](https://www.mysql.com/) stack. The performance, availability, and maintainability of its customer system have been significantly improved, which is the best practice of ShardingSphere’s heterogeneous migration.\n\n## Dangdang’s customer system\nDangdang’s customer system is mainly responsible for account registration, login, and privacy data maintenance. Its previous technology stack was based on PHP and SQL Server, which means a standard centralized architecture, as shown in the figure below.\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/lorqkp14im9bxbco4yuw.png)\n \n\nBefore the rebuild project started, several business modules of the customer system had encountered multiple problems and technical challenges, such as logical decentralization, low throughput, and high operation & maintenance costs.\n\nTo improve customers’ shopping experience, Dangdang’s technical team decided to optimize the business logic and underlying data architecture to achieve the availability, scalability, and comprehensive improvement of the customer system in multiple scenarios. The rebuild also introduced many technological innovations such as cross-data source double write, read/write splitting, intelligent gateway, and gray release.\n\nDangdang’s technical team completed the system rebuild within half a year, from demand design, sharding planning, logic optimization, and stress testing to its official launch.\n\nThe project used Java to reconstruct more than ten modules, build distributed database solutions through ShardingSphere & MySQL, and finally complete the online migration of heterogeneous databases. The project boasts the following highlights:\n\n- Reconstruct PHP business code using Java language.\n- Replace SQL Server with ShardingSphere & MySQL.\n- Complete online data migration of 350 million users.\n- Complete a seamless launch through the data double-write scheme.\n\n## Pain points & challenges\n\n**Business pain points**\nAt the business level, the registration and login logic of some modules of the customer system was scattered at different ends. This resulted in high maintenance costs, and the old technical architecture was limited in terms of performance improvement and high availability.\n\n- **Maintenance difficulty:** the registration and login logic of multiple platforms is scattered, so business maintenance is complicated.\n- **Limited performance:** the PHP & SQL Server, a centralized technical architecture, had insufficient throughput.\n- **Poor availability and security:** If the active/standby status of SQL Server changes, the subscription database becomes invalid and the reconfiguration takes a window of time. The security of SQL Server running on Windows Server is poor due to viruses, and the upgrading takes a long time (>30min) after the patch is installed.\n\n**Challenges**\n\n- **Data integrity:** the customer system involves data of more than 350 million users. It is necessary to ensure data consistency and integrity after migrating from SQL Server to MySQL.\n- **API transparency:** the API is transparent to the caller to ensure that the caller does not change and to minimize the change of interface.\n- **Seamless switch:** the business system must be seamlessly switched over without impact on business.\n- **Time is short:** the system will be blocked before and after “[618 (aka JD.com Day)](https://edition.cnn.com/2020/06/18/tech/jd-618-china-coronavirus-intl-hnk/index.html) and [11.11 (aka Singles Day)](https://en.wikipedia.org/wiki/Singles%27_Day)” (two online shopping festivals in China), so we need to switch it between the two shopping promotions in a limited window of time, and then undergo the tests to prepare for the 11.11 shopping festival.\n\n## Solutions\n**Overall planning**\nTo improve the maintainability, availability, and performance of the customer system, the R&D team reorganized the customer system architecture.\n\nAt the application layer, the goal was to unify the function logic of all terminals and improve business maintainability.\n\nAt the database layer, the centralized architecture was transformed into a distributed database architecture to improve performance and availability, which is exactly the open-source distributed solution built by ShardingSphere & MySQL.\n\n- **Application layer:** As Dangdang’s overall technology stack changed, its business development language changed from PHP to Java.\n- **Middleware:** As a mature open-source database middleware, ShardingSphere, was used to achieve data sharding.\n- **Database:** Multiple MySQL clusters were used to replace SQL Server databases.\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/bhrntsoik9fb41raqqbe.png)\n \nThe overall architecture design introduced multiple schemes, such as distributed primary-key generation strategy, shard management, data migration verification, and gray release.\n\n## Distributed primary-key generation strategy\n\nDistributed primary-key generation strategy is the first problem to be solved if database architecture is to be transformed from a centralized architecture to a distributed one based on middleware.\n\nDuring the system rebuild, we chose to build two or more database ID-generating servers. Each server had a `Sequence` table that records the current `ID` of each table. The step size of `ID` that increases in the `Sequence` table is the number of servers. The starting values are staggered so that the ID generation is hashed to each server node.\n\n## Implementing sharding (Apache ShardingSphere）\nDuring the customer system rebuild, database sharding was completed through Apache ShardingSphere, and the read/write splitting function was also enabled.\n\nDue to the requirements of the customer system for high concurrency and low latency, the access end chose [ShardingSphere-JDBC](https://shardingsphere.apache.org/document/current/en/overview/#shardingsphere-jdbc), which is positioned as a lightweight Java framework and provides additional services in Java’s JDBC layer.\n\nIt connects directly to the database via the client and provides services in the form of a `jar` package without additional deployment and dependence. It can be viewed as an enhanced version of the JDBC driver, fully compatible with JDBC and various [ORM](https://www.techopedia.com/definition/24200/object-relational-mapping--orm) frameworks.\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/6yyoi7xsp7qpayu1it5v.png)\n \n\n**Sharding:** ShardingSphere supports a complete set of sharding algorithms, including `modulo` operation, `hash`, `range`, `time`, and customized algorithms. Customers use the `modulo` sharding algorithm to split large tables.\n**Read-write splitting:** in addition to Sharding, ShardingSphere’s read/write splitting function is also enabled to make full use of [MHA](https://myheroacademia.fandom.com/wiki/Cluster) cluster resources and improve system throughput capacity.\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/chx6mgi5yfr2tqv86ho3.png)\n \n\n## Double-write & data synchronization\nData synchronization runs through the whole rebuild project, and the integrity and consistency of data migration are vital to the rebuild.\n\nThis example periodically synchronizes SQL Server’s historical data to MySQL based on Elastic-Job synchronization. During the database switchover, a backup scheme is used to double-write the database to ensure data consistency. The process consists of:\n\n**Step 1:** implement the double-write mechanism\n\nDisconnect link 1, get through links 2, 3, 4, and then 9, 10.\n\n**Step 2:** switch the login service\n\nDisconnect links 9,10, get through link 7 and disconnect link 5.\n\n**Step 3:** switch read service\n\nGet through link 8 and disconnect link 6.\n\n**Step 4:** cancel the double-write mechanism\n\nDisconnect link 2 and complete the switchover.\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/c2f4q20g116681261m9m.png)\n \n\nData verification is performed periodically on both the service side and the database side. Different frequencies are used in different time periods to sample or fully check data integrity. `COUNT/SUM` is also verified on the database side.\n\nCustomer system reconstruction adopts an apollo-based gray release. In the process of new login processing, configuration items are gradually released and sequential cutover within a small range is implemented to ensure the launch success rate. The rebuilt system architecture is shown in the following figure.\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/iru06ju7apswukev0wxc.png)\n \n\n## Advantages\nAfter the rebuild, the response speed of Dangdang’s customer system is significantly improved, and the daily operation & maintenance costs are also reduced.\n\nThe distributed solution provided by ShardingSphere plays a big part in this. The solution is suitable for various high-traffic Internet platform services, as well as e-commerce platforms and other data-processing systems.\n\n- **Performance improvement:** response speed increased by more than 20%.\n- **High availability:** RTO is reduced to less than 30s owing to the ShardingSphere & MySQL design.\n- **Easy to maintain:** business logic and database maintainability are significantly improved.\n- **Seamless migration:** complete online cutover of each module within 6 months, and window time is zero.\n\n## Conclusion\nThis is ShardingSphere’s second implementation by Dangdang, following the previous one we shared in the post “[Asia’s E-Commerce Giant Dangdang Increases Order Processing Speed by 30% — Saves Over Ten Million in Technology Budget with Apache ShardingSphere](https://shardingsphere.medium.com/asias-e-commerce-giant-dangdang-increases-order-processing-speed-by-30-saves-over-ten-million-113a976e0165)”.\n\nApache ShardingSphere provides strong support for enterprise systems, as the project strives for simplicity and perfection, to achieve simpler business logic and maximum performance.\n\n**Apache ShardingSphere Project Links:**\n\n[ShardingSphere Github](https://github.com/apache/shardingsphere/issues?page=1&q=is%3Aopen+is%3Aissue+label%3A%22project%3A+OpenForce+2022%22)\n\n[ShardingSphere Twitter](https://twitter.com/ShardingSphere)\n\n[ShardingSphere Slack](https://join.slack.com/t/apacheshardingsphere/shared_invite/zt-sbdde7ie-SjDqo9~I4rYcR18bq0SYTg)\n\n[Contributor Guide](https://shardingsphere.apache.org/community/cn/contribute/)","date":"2022-06-21","author":"SphereEx","excerpt":"E-commerce giant DangDang rebuilt its architecture & migrated 350M users with ShardingSphere & MySQL. RTO improved 60x, response speed by 20%, and introduced cross-data source double write, R/W split, and intelligent gateway innovations.","metaDescription":"Dangdang is a B2C-based e-commerce company in China offering a large selection of products both online and offline.","createdAt":"2022-09-26T06:47:46.510Z","updatedAt":"2022-09-27T06:55:21.087Z","publishedAt":"2022-09-27T06:55:21.084Z","locale":"en","caseStudiesType":{"data":{"id":9,"attributes":{"name":"Internet","createdAt":"2022-09-26T10:34:03.086Z","updatedAt":"2022-09-27T06:59:34.838Z","publishedAt":"2022-09-27T06:59:34.836Z","locale":"en"}}},"cover":{"data":{"id":336,"attributes":{"name":"Dangdang.png","alternativeText":"Dangdang.png","caption":"Dangdang.png","width":720,"height":432,"formats":{"thumbnail":{"name":"thumbnail_Dangdang.png","hash":"thumbnail_Dangdang_f80baa2e02","ext":".png","mime":"image/png","width":245,"height":147,"size":11.34,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_Dangdang_f80baa2e02.png"},"small":{"name":"small_Dangdang.png","hash":"small_Dangdang_f80baa2e02","ext":".png","mime":"image/png","width":500,"height":300,"size":28.45,"path":null,"url":"https://official-media.sphere-ex.com/small_Dangdang_f80baa2e02.png"}},"hash":"Dangdang_f80baa2e02","ext":".png","mime":"image/png","size":37.5,"url":"https://official-media.sphere-ex.com/Dangdang_f80baa2e02.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T06:48:03.795Z","updatedAt":"2022-09-26T06:48:03.795Z"}}},"metaLogo":{"data":{"id":351,"attributes":{"name":"dangdang logo.jpg","alternativeText":"dangdang logo.jpg","caption":"dangdang logo.jpg","width":500,"height":256,"formats":{"thumbnail":{"name":"thumbnail_dangdang logo.jpg","hash":"thumbnail_dangdang_logo_539787f05a","ext":".jpg","mime":"image/jpeg","width":245,"height":125,"size":5.42,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/thumbnail_dangdang_logo_539787f05a.jpg"}},"hash":"dangdang_logo_539787f05a","ext":".jpg","mime":"image/jpeg","size":12.11,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/dangdang_logo_539787f05a.jpg","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:37:49.195Z","updatedAt":"2022-10-12T05:57:41.631Z"}}},"localizations":{"data":[]}}},{"id":18,"attributes":{"feature":false,"title":"Apache ShardingSphere & Bilibili","content":"> To further understand application scenarios, and enterprises’ needs, and improve dev teams’ understanding of Apache ShardingSphere, our community launched the “Enterprise Visits”\n> series.\n\n\n[Apache ShardingSphere](https://shardingsphere.apache.org/)’s core contributor team was invited to the headquarters of [Bilibili](https://www.bilibili.com/) in Shanghai. Our PMC Chair, Zhang Liang, discussed with Bilibili’s tech team the e-commerce and digital entertainment verticals application scenarios, and the capabilities of different versions of ShardingSphere.\n\nWith the unprecedented growth of data volume and increasingly diversified data application scenarios, different platforms, businesses, and scenarios have resulted in database applications’ fragmentation. The challenges facing databases today are totally different from the ones they were conceived to solve. This is also reflected in the growth of Bilibili’s e-commerce business.\n\nFor over a decade, Bilibili has built an ecosystem community centered on users, creators, and content, while producing high-quality videos as well. With an increasing number of users, Bilibili has also gradually developed peripheral business ecosystems such as its subscription revenue model. The expansion of business lines and application scenarios brought great challenges for Bilibili’s tech team, especially in the management and application of backend data.\n\nDuring the visit, our community and Bilibili mainly discussed these three points:\n\n## The SQL warm-up feature jointly built with Bilibili:\nWhen Bilibili queries goods and orders with batch priority in its business, it is often necessary to manually warm-up SQL during the initial linking process to improve overall performance. However, in the process of manual warm-up, parameters of different lengths cannot be used as a SQL to warm-up due to the use of `foreach` syntax of the [Mybatis](https://mybatis.org/mybatis-3/index.html) framework.\n\nApache ShardingSphere can see the SQL execution plan through the `preview`, thus warming up the SQL. In the future, we plan to provide a separate SQL warm-up interface and merge the SQL warm-up time into the startup time. ShardingSphere will start on its own after SQL is warmed up.\n\n[Apache ShardingSphere](https://shardingsphere.apache.org/) is an open-source project driven by the needs of the community. Currently, many capabilities’ development is driven by specific user requirements, which are fed back to the community after development and gradually integrated into Apache ShardingSphere.\n\nTherefore, the ShardingSphere community invited Bilibili’s tech team to get involved in terms of the SQL warm-up feature.\n\nWith the expansion of Apache ShardingSphere’s application scenarios, there is a higher expectation for ShardingSphere’s capabilities to adapt to a variety of special business scenarios. In the previous “Enterprise Visits” series, the ShardingSphere community recognized that even though it has more than 100 feature modules, enterprises from various verticals have different expectations from ShardingSphere.\n\n## Bilibili’s circuit breaker automation in traffic spike scenarios:\n\nWith the growth of Bilibili’s e-commerce scale, as is often the case with many e-commerce businesses, started to adopt front-end strategies such as cash rebates and flash deals in order to improve user retention.\n\nOn the backend, however, when the rebate performance of the warehouse is equal to the number of concurrent transactions while exceeding its own connection number limit, it can only be stopped through manual disconnection via DBA based on the number of SKU at the business level. Manual intervention is inefficient and consumes DBAs’ energy, so [ShardingSphere-Proxy](https://shardingsphere.apache.org/document/current/en/quick-start/shardingsphere-proxy-quick-start/) is expected to provide an automatic circuit breaker capability.\n\nHowever, the flash deal scenario is too demanding, so [Redis](https://redis.io/) still represents a better choice. Achieving automation simply requires setting rules and thresholds.\n\nSinking data to Proxy can also achieve the capabilities of Redis, but no matter how the upper layer changes, the bottom of the database will not change. Therefore, it is better for DBAs to take the initiative to operate the circuit breaker mechanism. Otherwise, after setting a threshold, if the connection between the application and the database has a slight timeout, a large number of transactions will be cut off instantly, which can easily cause a business outage.\n\nCurrently, a better way is to dig out all kinds of key information and display it on the visual interface based on Proxy, to facilitate the real-time comparison and operation of DBA instead of achieving automation.\n\n## The Apache ShardingSphere registry:\nIn the Apache ShardingSphere architecture, the registry provides distributed governance capabilities and is fully open to users since its computing node ([Shardingsphere-proxy](https://shardingsphere.apache.org/document/current/en/quick-start/shardingsphere-proxy-quick-start/)) is stateless without data storage capability. Therefore, user accounts and authorization information need to be stored in the registry.\n\nConcurrently, with the help of the registry, Apache ShardingSphere can distribute information to multiple computing nodes in the cluster in real-time, greatly reducing maintenance costs for users when using the cluster and improving management efficiency.\n\nIn cluster mode, Apache ShardingSphere integrates third-party registry components [ZooKeeper](https://zookeeper.apache.org/) and [Etcd](https://etcd.io/) to achieve metadata and configuration sharing in the cluster environment. At the same time, with the help of the notification and coordination ability of the registry, it ensures the real-time synchronization of the cluster when the shared data changes. And the business will not be aware of changes from the registry.\n\n## Q&A with Bilibili\n**Q: Is there a performance loss when using JDBC to connect to the governance center?**\n\nA: No. It only connects to the governance center during initialization, and the governance center will send a push when there is a change.\n\n**Q: How does [Sysbench](https://wiki.gentoo.org/wiki/Sysbench) conduct a stress test on Apache ShardingSphere?**\n\nA: The two deployment types of Apache ShardingSphere, JDBC and Proxy, are both tested by Sysbench. ShardingSphere has a newly designed Sysbench-like Java program on the JDBC end that can be used to conduct a pressure test to JDBC and Proxy. It can also ensure that the official Sysbench and our Java program share the same standard.\n\n**Q: Can ShardingSphere converge the connection number?**\n\nA: ShardingSphere-Proxy can converge the connection number, but it will definitely lead to performance loss.\n\n**Q: Can Proxy identify slow SQL?**\n\nA: Currently, the open-source version doesn’t support this function, because most users of the open-source version are Internet enterprises with a low tolerance for performance loss. Thus, the number of probes is relatively small.\n\n**Q: Does [ElasticJob](https://shardingsphere.apache.org/elasticjob/) belong to Apache ShardingSphere?**\n\nA: ElasticJob is currently used as the migration tool by Apache ShardingSphere. Additionally, ElasticJob can also be used for liveness probes.\n\n**Q: Are Internet enterprises using Proxy on a large scale?**\n\nA: Most Internet users choose the mixed deployment model, using JDBC for development and better performance, and Proxy for management. Financial customers prefer to use Proxy because they can take Proxy as a database for unified management without additional learning costs.\n\n**Q: We are currently using ShardingSphere version 4.1.1, what does it support for transactions?**\n\nA: Both versions 4.11 and 5.1.0 support [XA](https://docs.oracle.com/database/121/TTCDV/xa_dtp.htm#TTCDV327) distributed transaction management. We plan to develop the global transaction manager ([GTM](https://docs.oracle.com/cd/E17276_01/html/programmer_reference/xa_build.html)) which is scheduled to start in the second half of this year.\n\n## Get in touch with us\nIf you have applied Apache ShardingSphere solutions in your business or if you want to quickly understand and introduce the Apache ShardingSphere 5.X ecosystem to your business, you’ll probably like for someone from our community to help you out and share the Apache ShardingSphere technology with your team.\n\nFeel free to reach out to us on one of our official community channels, such as Twitter or Slack.\n\nIf we both agree that ShardingSphere is suitable for your business scenarios, our community team will be happy to connect with you and your engineers to take their questions.\n\n**Apache ShardingSphere Project Links:**\n\n[ShardingSphere Github](https://github.com/apache/shardingsphere/issues?page=1&q=is%3Aopen+is%3Aissue+label%3A%22project%3A+OpenForce+2022%22)\n\n[ShardingSphere Twitter](https://twitter.com/ShardingSphere)\n\n[ShardingSphere Slack](https://join.slack.com/t/apacheshardingsphere/shared_invite/zt-sbdde7ie-SjDqo9~I4rYcR18bq0SYTg)\n\n[Contributor Guide](https://shardingsphere.apache.org/community/cn/contribute/)","date":"2022-06-07","author":"SphereEx","excerpt":"The ShardingSphere community visited one of its enterprise users: the major Chinese video-on-demand streaming platform - Bilibili. Originally known for anime, comics & games, Bilibili has since expanded - as did its total users.\n","metaDescription":"Bilibili is an online entertainment platform featuring videos, live broadcasting, and mobile games.","createdAt":"2022-09-26T06:51:57.325Z","updatedAt":"2022-09-27T06:59:09.655Z","publishedAt":"2022-09-27T06:59:09.651Z","locale":"en","caseStudiesType":{"data":{"id":9,"attributes":{"name":"Internet","createdAt":"2022-09-26T10:34:03.086Z","updatedAt":"2022-09-27T06:59:34.838Z","publishedAt":"2022-09-27T06:59:34.836Z","locale":"en"}}},"cover":{"data":{"id":337,"attributes":{"name":"bilibili.jpeg","alternativeText":"bilibili.jpeg","caption":"bilibili.jpeg","width":1024,"height":450,"formats":{"thumbnail":{"name":"thumbnail_bilibili.jpeg","hash":"thumbnail_bilibili_2865813b49","ext":".jpeg","mime":"image/jpeg","width":245,"height":108,"size":10.11,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_bilibili_2865813b49.jpeg"},"large":{"name":"large_bilibili.jpeg","hash":"large_bilibili_2865813b49","ext":".jpeg","mime":"image/jpeg","width":1000,"height":439,"size":86.56,"path":null,"url":"https://official-media.sphere-ex.com/large_bilibili_2865813b49.jpeg"},"medium":{"name":"medium_bilibili.jpeg","hash":"medium_bilibili_2865813b49","ext":".jpeg","mime":"image/jpeg","width":750,"height":330,"size":56.83,"path":null,"url":"https://official-media.sphere-ex.com/medium_bilibili_2865813b49.jpeg"},"small":{"name":"small_bilibili.jpeg","hash":"small_bilibili_2865813b49","ext":".jpeg","mime":"image/jpeg","width":500,"height":220,"size":30.65,"path":null,"url":"https://official-media.sphere-ex.com/small_bilibili_2865813b49.jpeg"}},"hash":"bilibili_2865813b49","ext":".jpeg","mime":"image/jpeg","size":90.83,"url":"https://official-media.sphere-ex.com/bilibili_2865813b49.jpeg","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T06:52:45.832Z","updatedAt":"2022-09-26T06:52:45.832Z"}}},"metaLogo":{"data":{"id":352,"attributes":{"name":"bilibili logo.jpg","alternativeText":"bilibili logo.jpg","caption":"bilibili logo.jpg","width":1050,"height":758,"formats":{"thumbnail":{"name":"thumbnail_bilibili logo.jpg","hash":"thumbnail_bilibili_logo_560e3483f4","ext":".jpg","mime":"image/jpeg","width":216,"height":156,"size":4.21,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_bilibili_logo_560e3483f4.jpg"},"large":{"name":"large_bilibili logo.jpg","hash":"large_bilibili_logo_560e3483f4","ext":".jpg","mime":"image/jpeg","width":1000,"height":722,"size":27.06,"path":null,"url":"https://official-media.sphere-ex.com/large_bilibili_logo_560e3483f4.jpg"},"medium":{"name":"medium_bilibili logo.jpg","hash":"medium_bilibili_logo_560e3483f4","ext":".jpg","mime":"image/jpeg","width":750,"height":541,"size":18.89,"path":null,"url":"https://official-media.sphere-ex.com/medium_bilibili_logo_560e3483f4.jpg"},"small":{"name":"small_bilibili logo.jpg","hash":"small_bilibili_logo_560e3483f4","ext":".jpg","mime":"image/jpeg","width":500,"height":361,"size":11.77,"path":null,"url":"https://official-media.sphere-ex.com/small_bilibili_logo_560e3483f4.jpg"}},"hash":"bilibili_logo_560e3483f4","ext":".jpg","mime":"image/jpeg","size":27.87,"url":"https://official-media.sphere-ex.com/bilibili_logo_560e3483f4.jpg","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:40:54.591Z","updatedAt":"2022-09-26T08:40:54.591Z"}}},"localizations":{"data":[]}}},{"id":19,"attributes":{"feature":false,"title":"Apache ShardingSphere & Huimin: Huimin’s Transaction Middle Platform Architecture","content":"The FMCG (fast moving consumer-goods) industry can no longer meet current development requirements. The traditional way of only upgrading the end of the supply chain cannot achieve high-quality business development. [Huimin](https://huimin.cn/) focuses on order fulfillment and delivery service for over one million community supermarkets, transforming and upgrading from serving retail terminal stores to promoting the digitization of the whole FMCG industry chain.\n\nIt is committed to building a B2B2C data closed-loop operating system, providing brand owners, distributors, and community supermarkets with transformed and upgraded solutions for the whole chain covering purchasing, marketing, operation, and distribution. All these commitments cannot be fulfilled without its powerful “middle platform” system.\n\n## Introducing the “middle platform”\nHuimin began to transferred [PHP](https://www.php.net/) technology stack to the Java technology stack, and shifted to microservices from 2016 to 2017, and started to build the “middle platform” in 2018.\n\nThe phase-one project of its transaction middle platform went live in early 2021, and phase-two in March 2022.\n\nOwing to business growth and various changes, Huimin launched a strong “middle platform” development strategy to improve efficiency and reduce costs. Measures were taken to reduce the coupling degree between systems, increase scalability, and ensure low latency while extracting business commonalities. At the beginning of 2021, the company finihsed rebuilding the previous order management system (OMS) and launched a packaged business capability project for a transaction middle platform centered on the transaction process of orders, which is divided into the following four core modules: order status flow system, order management system, order fulfillment system and order cost calculation system. At the same time, based on business stratification, the project is also split into an order query system, report system, and order management backend.\n\nTo synergize technical and business requirements during the rebuilding process, the go-live process was divided into two phases: application system splitting and data splitting.\n\n## System splitting\nR&D teams having to frequently adjust order management system and adapt it to multiple business lines may cause inefficiencies. In response to these problems, the following principles were formulated to split and rebuild the system:\n\n1. Avoid the logic coupling of different lines of business. Different business lines should contain and minimize influence on each other to reduce logic complexity.\n2. Responsibilities are divided according to the order lifecycle to ensure that each system is independent with an exclusive responsibility.\n3. Read/write splitting. Ensure the stability of core functions and avoid the impact of frequent iterations on core functions.\n4. [ElasticSearch](https://www.elastic.co/) was introduced to solve the external index problem, and reduce database index pressure.\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/1223xl1pboe6emvrjian.png)\n \n\n## Data splitting\nThe purpose of data splitting is to reduce the maintenance pressure of a single table. When the amount of data reaches tens of millions, the index and field maintenance of database tables will have a great impact on the online environment. Before splitting a table, you have to consider the following questions:\n\n1. Splitting rules: what tables are data-intensive? What about the number of splits?\n2. Data read/write: how to solve the read/write problem after splitting tables?\n3. Go-live solution: how to go live smoothly?\n\n**Splitting rules**\n\nDuring a 3 years iteration period, the number of splits = ((amount of data added per month * 36) + Amount of historical data) / maximum amount of data in a single table.\n\nWhen we use the [InnoDB](https://en.wikipedia.org/wiki/InnoDB) engine for DDL operation to maintain [Alibaba Cloud](https://www.alibabacloud.com/?spm=a2796.143805.6791778070.2.b8fa46e3JLecKh) RDS MySQL, we come up with the following results:\n\nWhen the amount of data is less than 5 million, the execution time is about tens of seconds. Between 5 million and less than 10 million, the execution time is about 500 seconds; between 10 million and less than 50 million, the execution time is about 1000 seconds; and when more than 50 million, the execution time is more than 2000 seconds.\n\nThe upper limit for splitting a single table depends on the impact of a single table operation on business. Of course, some solutions can tackle the problem of table lock resulting from DDL operation. For example, dual-table synchronization and changes in table names can reduce the impact to within seconds.\n\nThe split number of the library table is preferably 2 to the power of N, which is conducive to later horizontal scaling.\n\n**Technology selection**\n\nAfter the data table is split, data hashing and query issues need to be resolved. As a SME (small & medium-sized enterprise), the cost of developing a set of sharding middleware is too high, and it will cause various risks in the early stage. A reasonable solution is to adopt a set of open source database sharding middleware.\n\n**[Apache ShardingSphere](https://shardingsphere.apache.org/)**\n\nApache ShardingSphere is an open source ecosystem that allows you to transform any database into a distributed database system. The project includes a JDBC, a Proxy and Sidecar (Planning), which can be deployed independently and support mixed deployment. They all provide functions of data scale-out, distributed transactions and distributed governance.\n\nApache ShardingSphere 5.x introduces the concept of [Database Plus](https://faun.pub/whats-the-database-plus-concepand-what-challenges-can-it-solve-715920ba65aa), and the new version focuses on a plugin oriented architecture. Currently, functions including data sharding, read/write splitting, data encryption, shadow library pressure testing, as well as SQL and protocol support such as [MySQL](https://www.mysql.com/), [PostgreSQL](https://www.postgresql.org/), [SQL Server](https://www.microsoft.com/en-us/sql-server/sql-server-downloads), [Oracle](https://www.oracle.com/index.html), etc. are woven into the project through plugins, allowing developers to customize their own unique systems.\n\nWhy Huimin chose Apache ShardingSphere:\n\n1. Apache ShardingSphere has a variety of functions and can solve current challenges with strong scalability.\n\n2. The Apache ShardingSphere community is very active, which means the possibility to find support quickly.\n\n3. The company adopts the SpringCloud technology stack which is convenient for integration and can lower costs.\n\n4. Its performance can fully support Huimin’s existing business.\n\nShardingSphere supports the following 3 modes:\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/19343yqtstyzsmvtu69j.png)\n \n\nCurrently, Huimin’s server-side technology stack only involves Java language, so there is no need to consider heterogeneous scenarios. Taking flexibility, code intrusion and deployment cost into consideration, the implementation of [ShardingSphere-JDBC](https://shardingsphere.apache.org/document/current/en/overview/#shardingsphere-jdbc) wasfinally selected.\n\n**Technology implementation**\n\nThe most complicated part of the process is actually the go-live part, not the plugin integration part as would someone expect.\n\nTo achieve non-perception and rollback, we divided the go-live process into the following steps, as shown in the figure:\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/dyrbcvaql5nhnw4m0xns.png)\n \n\n**Step 1:** Processing all the data needs to ensure the data consistency between the old and new libraries. Since there currently is no synchronization tool supporting data sharding policy, we developed a data synchronization tool to support the configuration of the sharding policy.\n\n**Step 2:** Process incremental data to keep data consistency between the old and new libraries. Our implementation scheme here was to use the open source component, to listen to the binlog of the database and synchronize the database changes to the message queue, and then the data synchronization tool listens to the messages and writes them into the new library.\n\nThe architecture scheme is not complicated, but great importance was attached to the data consistency in this process. Incremental data cannot be lost, and single-line data should be written in sequential order to prevent ABA problems.\n\n**Step 3:** Process the gray-release of read-traffic. At this point, we were writing an old library and reading a new library in the database. In this process, we should paid attention to the sensitivity of business to data consistency, because the delay from an old library to a new library is within seconds. Reading an old library is needed when we come across some sensitive scenarios requiring high consistency.\n\n**Step 4:** Switch all read-traffic of the application to the new library, maintaining the status of writing the old library and reading the new library.\n\n**Step 5:** Processing write-traffic didn’t take gray-release scenarios into consideration in order to reduce the complexity of the program. Of course, the grayscale scheme can also achieve the same effect. We supported the rollback strategy by writing data from the new library back to the old library, and then uniformly transferring all traffic to the new library within one release.\n\nSo far, the whole process of release and going live has been completed. Due to the low cost of completing the transformation with the help of ShardingSphere-JDBC middleware to rewrite SQL and merge results, there was almost no code intrusion and going live went smoothly.\n\nThere are also some issues that you may need to be aware of when accessing the ShardingSphere-JDBC scheme:\n\n1. ShardingSphere-JDBC supports a default processing for data insertion logic. If there is no shard column in SQL, the same data will be inserted into each shard table.\n\n2. ShardingSphere-JDBC parses only the logil table of the first SQL for batches of SQL semantic analysis, which results in an error. This problem has been reported and should be fixed in the near future. Additionally, a detailed SQL sample is provided, which lists in detail the scope of support.\n\n3. ShardingSphere connection mode selection. The number of database connections visited by businesses should be limited for the sake of resource control. It can effectively prevent a service operation from occupying too many resources, which depletes the database connection resources and affects the normal access of other services. Especially in the case of many sub-tables in a database instance, a logical SQL without shard keys will produce a large number of actual SQL that falls on different tables in the same library. If each actual SQL occupies a separate connection, a query will undoubtedly occupy too many resources.\n\nFrom the perspective of execution efficiency, maintaining a separate database connection for each shard query can make more effective use of multi-threading to improve execution efficiency. Building a separate thread for each database connection allows I/O consumption to be processed concurrently, maintains a separate database connection for each shard, and avoids premature loading of query result data into memory. A separate database connection can hold a reference to the cursor location of the query result set and move the cursor when it needs to fetch the corresponding data.\n\nThe result merging method by moving the result set cursor down is called streaming merge. It does not have to load all the result data into memory, which can effectively save memory resources and reduce the frequency of garbage collection. If each shard query cannot be guaranteed to have a separate database connection, the current query result set needs to be loaded into memory before reusing the database connection to obtain the query result set of the next shard table. Therefore, even if streaming merge could be used, it would degenerate into a memory merge in this scenario.\n\nIt is a problem for the ShardingSphere execution engine to balance the relationship between the control and protection of database connection resources and saving middleware memory resources by adopting a better merging mode. Specifically, if a SQL needs to operate on 200 tables under a database instance after its sharding in ShardingSphere, whether to create 200 connections and execute them concurrently or create one connection and execute it in sequence? How to choose between efficiency and resource control?\n\nIn response to the above scenario, ShardingSphere provides a solution. It introduces the concept of Connection Mode and divides it into `MEMORY_STRICTLY` mode and `CONNECTION_STRICTLY` mode.\n\n`MEMORY_STRICTLY`\n\nThe premise of using this pattern is that ShardingSphere does not limit the number of database connections required by an operation. If the actual SQL executed needs to operate on 200 tables in a database instance, a new database connection is created for each table and processed concurrently through multi-threading to maximize the execution efficiency. When SQL meets the conditions, streaming merge is preferred to prevent memory overflow or frequent garbage collection.\n\n`CONNECTION_STRICTLY`\n\nThe premise for using this pattern is that ShardingSphere strictly limits the number of database connections required for an operation. If the actual SQL executed needs to operate on 200 tables in a database instance, only one single database connection is created and its 200 tables are processed sequentially. If shards in an operation are scattered across different databases, multi-threading is still used to handle operations on different libraries, but only one database connection is created for each operation of each library. The practice prevents the problem of consuming too many database connections for one request. This pattern adopts memory merge all the time.\n\nThe `MEMORY_STRICTLY` mode is applicable for OLAP operations and can improve system throughput by relaxing limits on database connections. The `CONNECTION_STRICTLY` mode applies to OLTP operations, which usually have shard keys and are routed to a single shard. Therefore, it is a wise choice to strictly control database connections to ensure that database resources in an online system can be used by more applications.\n\nWe found that in the `MEMORY_STRICTLY` mode, the operation would become I/O-intensive due to the cache buffer loading strategy of MySQL InnoDB engine, resulting in SQL timeouts. Its solution is to add another layer of processing without changing database resources. If there is no shard key, verify the shard key in the external index, and then use the shard key to perform database retrieval.\n\n## ShardingSphere Benefits\n**1. Performance improvement**\nThrough architecture rebuilding, it can effectively control the amount of single table data, and greatly reduce the slow SQL, down nearly 50%.\n\n**2. Save R&D resources and lower cost**\n\nThe introduction of Apache ShardingSphere does not require the re-development of sharding components, which reduces the R&D cost and lowers the risks.\n\n**3. Strong scalability**\n\nApache ShardingSphere has good scalability in terms of data encryption, distributed transactions, shadow library and other aspects.\n\n**Conclusion**\nIn his book [Building Microservices, Sam Newman](https://samnewman.io/books/building_microservices/) writes, “ Our requirements shift more rapidly than they do for people who design and build buildings — as do the tools and techniques at our disposal. The things we create are not fixed points in time. Once launched into production, our software will continue to evolve as the way it is used changes. For most things we create, we have to accept that once the software gets into the hands of our customers we will have to react and adapt, rather than it being a never-changing artifact. Thus, our architects need to shift their thinking away from creating the perfect end product, and instead focus on helping create a framework in which the right systems can emerge, and continue to grow as we learn more (Sam Newman, 2015: 15).” Apache ShardingSphere is doing exactly what the book says, and is definitely on a promising path.\n\n## Author\n**Zhai Yang**\n\nSoftware architect at Zhongshang Huimin E-Commerce platform team.\n\nParticipated in the 0–1 process of middle platform construction, mainly responsible for the R&D and building the transaction and commodity middle platform and the search platform.","date":"2022-05-17","author":"SphereEx","excerpt":"HuiMin does order fulfillment for over 1 million supermarkets. With ShardingSphere HuiMin shifted to microservices & built a new \"middle platform\" architecture with packaged business capability for order transaction processing.","metaDescription":"Huimin is a B2B e-commerce platform that focuses on small-scale supermarkets that provides community supermarket websites for communities.","createdAt":"2022-09-26T07:01:29.374Z","updatedAt":"2022-09-27T06:59:59.417Z","publishedAt":"2022-09-27T06:59:59.413Z","locale":"en","caseStudiesType":{"data":{"id":2,"attributes":{"name":"E-commerce","createdAt":"2021-12-11T23:02:08.395Z","updatedAt":"2021-12-11T23:02:11.020Z","publishedAt":"2021-12-11T23:02:10.947Z","locale":"en"}}},"cover":{"data":{"id":338,"attributes":{"name":"Zhongshanghuimin.png","alternativeText":"Zhongshanghuimin.png","caption":"Zhongshanghuimin.png","width":720,"height":339,"formats":{"thumbnail":{"name":"thumbnail_Zhongshanghuimin.png","hash":"thumbnail_Zhongshanghuimin_6ceb4a8b12","ext":".png","mime":"image/png","width":245,"height":115,"size":19.04,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_Zhongshanghuimin_6ceb4a8b12.png"},"small":{"name":"small_Zhongshanghuimin.png","hash":"small_Zhongshanghuimin_6ceb4a8b12","ext":".png","mime":"image/png","width":500,"height":235,"size":60.18,"path":null,"url":"https://official-media.sphere-ex.com/small_Zhongshanghuimin_6ceb4a8b12.png"}},"hash":"Zhongshanghuimin_6ceb4a8b12","ext":".png","mime":"image/png","size":73.86,"url":"https://official-media.sphere-ex.com/Zhongshanghuimin_6ceb4a8b12.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T07:01:49.173Z","updatedAt":"2022-09-26T07:01:49.173Z"}}},"metaLogo":{"data":{"id":353,"attributes":{"name":"zhongshanghuimin logo.png","alternativeText":"zhongshanghuimin logo.png","caption":"zhongshanghuimin logo.png","width":608,"height":430,"formats":{"thumbnail":{"name":"thumbnail_zhongshanghuimin logo.png","hash":"thumbnail_zhongshanghuimin_logo_978897c9d6","ext":".png","mime":"image/png","width":221,"height":156,"size":9.77,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/thumbnail_zhongshanghuimin_logo_978897c9d6.png"},"small":{"name":"small_zhongshanghuimin logo.png","hash":"small_zhongshanghuimin_logo_978897c9d6","ext":".png","mime":"image/png","width":500,"height":354,"size":31.4,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/small_zhongshanghuimin_logo_978897c9d6.png"}},"hash":"zhongshanghuimin_logo_978897c9d6","ext":".png","mime":"image/png","size":12.04,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/zhongshanghuimin_logo_978897c9d6.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:42:11.374Z","updatedAt":"2022-10-12T05:56:46.710Z"}}},"localizations":{"data":[]}}},{"id":20,"attributes":{"feature":false,"title":"Apache ShardingSphere & Zhuanzhuan: Zhuanzhuan’s Transaction System with 100s of Millions of Records","content":"## Background and Challenges\n[Zhuanzhuan](https://www.zhuanzhuan.com/index.html) is an internet platform that allows it users to sell their second-hand stuff — sort of an eBay of the East. Its business had been booming, and with it the ordering system started to face increasing performances challenges. The order database is the cornerstone of the system, and its performance should not be underestimated.\n\n**Challenges:**\n\n- During promotions and special discount periods the burden on databases is heaby with tens of thousands of single database queries per second (qps) taking up huge database resources, and causing a significant reduction in write performance.\n- Increased data pressure, with a single database containing several large tables with hundreds of millions of data, which challenges the capacity limit of the server.\n- Overwhelming data volume, and data backup and recovery take a long time, posing high risks of data loss in extreme cases.\n\n## Why ShardingSphere?\nIn the beginning, ZhuanZhuan’s team took adjustment measures to ease the database pressure. Exmaples include:\n\n**- Optimized major transactions, reduced transactions, and even eliminated transactions**\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/jgvh0xy3l3xvjkbn7c4j.png)\n \n\nWe adjusted the original transaction order by putting table generation, the core step at the end, and keeping the transaction only in the order primary database. When the operation of the main table was abnormal, dirty reads were allowed on other order-related tables.\n\n**- Order data cache**\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/8ueib5czc3uni28w0yk5.png)\n \n\nData consistency was the trickiest part of the cache. As order data involved account settlements and commission, non-real-time and inconsistent data would cause serious accidents.\n\nStrictly keeping cache data consistency would complex coding and reduce system concurrency. Therefore, we made some compromises on cache plans:\n\n1. Allowing direct query when cache failed.\n2. Adding version serial number, and querying the latest version’s data to ensure real-time data.\n3. Complex queries were conducted by [Elasticsearch (ES)](https://www.elastic.co/) and primary and secondary separation, and for some large tables, we adopted hot and cold data separation.\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/k92mmvjo5y2ldbkskqc6.png)\n \n\nThrough these optimizations, database pressure was eased. However, it still seemed overwhelming under high concurrency scenarios, such as discount season.\n\nTo fundamentally solve the performance problem of order database, ZhuanZhuan decided to adopt data sharding (database and table splitting) on the `order` database so that we wouldn’t have to worry about order capacity in the future 3–5 years.\n\nZhuangzhuang chose [ShardingSphere](https://shardingsphere.apache.org/) after comparing the efficiency, stability, learning cost and etc. of different data sharding components.\n\nAdvantages of ShardingSphere:\n\n- It provides standardized data sharding, distributed transactions and database governance, and it’s applicable in a variety of situations such as Java isomorphism, heterogeneous language and cloud native.\n- It has flexible sharding strategies, supporting multiple sharding methods.\n- It’s easy to integrate with other components and has a low level of transaction intrusions.\n- It has extensive documentation and an active community.\n\nShardingSphere initiated the Database Plus concept and adopts a plugin oriented architecture where all modules are independent of each other, allowing each to be used individually or flexibly combined.\n\nIt consists of three products, namely [JDBC](https://shardingsphere.apache.org/), [Proxy](https://shardingsphere.apache.org/document/current/en/overview/#shardingsphere-proxy) and [Sidecar ](https://shardingsphere.apache.org/document/current/en/overview/#shardingsphere-sidecartodo)(Planning), which supports both independent and hybrid deployment.\n\nBelow is a feature comparison of the three products:\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/7517wpdgk6gmgb0s8202.png)\n \n\nBy comparison, and considering the high order concurrency, we chose ShardingSphere-JDBC as our data sharding middleware.\n\nShardingSphere-JDBC is a lightweight Java framework, proving extra service at the JDBC layer. It directly connects to the database by the client-side, provides services by Jar package, and requires no extra deployment and reliance. It can be seen as an enhanced JDBC driver, fully compatible with JDBC and other Object-Relational Mapping(ORM) frameworks.\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/luou74i1f1gpgowequk9.png)\n \n\n## Key Points in Project Implementation\n\n**- Sharding Key**\nThe current order ID is generated by `timestamp+user identification code+machine code+incremental sequence`. The user identification code is taken from bits 9 to 16 of the buyer ID, a true random number when the user ID is generated, and is thus suitable as a sharding key.\n\nChoosing user identification code as the sharding key has some advantages:\n\n- The data can be distributed as evenly as possible to each database and table.\n- Specific sharding locations can be quickly located either by order ID or user ID.\n- Data of the same buyer can be distributed to the same databases and tables, facilitating the integrated query of the buyer information.\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/jkilhtbnnwk9gcdn7l5x.png)\n \nThe sharding strategy: we adopt 16 databases and 16 tables. User identification codes are used to split databases, and higher 4 bits are used to split tables.\n\n**- Data Migration between Old and New Databases**\nThe migration must be online, and downtime migration cannot be accepted, as there will be new data writes during the migration process.\n\nThe data should be intact, and the migration process should be insensible to the client-side. After the migration, data in the new database should be consistent with the ones in the old databases.\n\nThe migration should allow rollback, so that when a problem occurs during the migration process, it should be able to roll back to the source database without impacting system availability.\n\nData migration steps are as follows: dual writes-> migrate historical data-> verify-> old database offline.\n\n## Effects and Benefits\n\n- It solves the problem of single database capacity limit.\n- The data volume of a single database and table is greatly reduced after sharding. The data volume of a single table is reduced from nearly a hundred million level to several millions level, which greatly improves the overall performance.\n- It reduces the risk of data losses due to oversized single databases and tables in extreme cases and eases the pressure of operation and maintenance.\nThe following is a comparison of the number of interface calls of the order placement service and the time consumed by the interface during two promotion and discount periods:\n\n**Promotion before adopting ShardingSphere**\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/l8d6f5tchgq7uipwbip0.png)\n \n\n**Promotion after adopting ShardingSphere**\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/3s9mtqhr0kpkecyz94kl.png)\n \n\n## Summary\nShardingSphere simplifies the development of data sharding with its well-designed architecture, highly flexible, pluggable and scalable capabilities, allowing R&D teams to focus only on the business itself, thus enabling flexible scaling of the data architecture.\n\n**Apache ShardingSphere Project Links:**\n\n[ShardingSphere Github](https://github.com/apache/shardingsphere/issues?page=1&q=is%3Aopen+is%3Aissue+label%3A%22project%3A+OpenForce+2022%22)\n\n[ShardingSphere Twitter](https://twitter.com/ShardingSphere)\n\n[ShardingSphere Slack](https://join.slack.com/t/apacheshardingsphere/shared_invite/zt-sbdde7ie-SjDqo9~I4rYcR18bq0SYTg)\n\n[Contributor Guide](https://shardingsphere.apache.org/community/cn/contribute/)","date":"2022-04-29","author":"SphereEx","excerpt":"Sales promotions drive traffic, and put great pressure on databases. Here's how \n ShardingSphere solves an e-commerce platform's data challenges created by sales promotion days.","metaDescription":"Zhuanzhuan is an online marketplace on which users can buy and sell secondhand goods.","createdAt":"2022-09-26T07:06:47.526Z","updatedAt":"2022-09-27T07:00:05.205Z","publishedAt":"2022-09-27T07:00:05.202Z","locale":"en","caseStudiesType":{"data":{"id":9,"attributes":{"name":"Internet","createdAt":"2022-09-26T10:34:03.086Z","updatedAt":"2022-09-27T06:59:34.838Z","publishedAt":"2022-09-27T06:59:34.836Z","locale":"en"}}},"cover":{"data":{"id":339,"attributes":{"name":"zhuanzhuan.png","alternativeText":"zhuanzhuan.png","caption":"zhuanzhuan.png","width":720,"height":534,"formats":{"thumbnail":{"name":"thumbnail_zhuanzhuan.png","hash":"thumbnail_zhuanzhuan_a2afd90ff3","ext":".png","mime":"image/png","width":210,"height":156,"size":11,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_zhuanzhuan_a2afd90ff3.png"},"small":{"name":"small_zhuanzhuan.png","hash":"small_zhuanzhuan_a2afd90ff3","ext":".png","mime":"image/png","width":500,"height":371,"size":36.3,"path":null,"url":"https://official-media.sphere-ex.com/small_zhuanzhuan_a2afd90ff3.png"}},"hash":"zhuanzhuan_a2afd90ff3","ext":".png","mime":"image/png","size":47.13,"url":"https://official-media.sphere-ex.com/zhuanzhuan_a2afd90ff3.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T07:07:25.422Z","updatedAt":"2022-09-26T07:07:25.422Z"}}},"metaLogo":{"data":{"id":354,"attributes":{"name":"zhuanzhuan logo.png","alternativeText":"zhuanzhuan logo.png","caption":"zhuanzhuan logo.png","width":130,"height":130,"formats":null,"hash":"zhuanzhuan_logo_fd59d35610","ext":".png","mime":"image/png","size":1.85,"url":"https://official-media.sphere-ex.com/zhuanzhuan_logo_fd59d35610.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:45:59.332Z","updatedAt":"2022-09-26T08:45:59.332Z"}}},"localizations":{"data":[]}}},{"id":27,"attributes":{"feature":false,"title":"Apache ShardingSphere & openGauss: Breaking the Distributed Database Performance Record with 10 Million tpmC","content":"Our open source community has cooperated Huawei’s openGauss to build a distributed solution with [Apache ShardingSphere](https://shardingsphere.apache.org/) and [openGauss](https://opengauss.org/en/).\n\nWe tested performance together with openGauss on 16 servers for more than one hour. The results were great: our joint solution broke the performance bottleneck of a single machine with a benchmark result of 10 million transactions per minute (tpmC) on average.\n\n\n## Breaking the 10 Million tpmC Barrier\nIn this test, the openGauss community ran this TPC-C testing on [BenchmarkSQL](https://www.postgresql.org/message-id/CAGBW59f9q2Y4v-B3D8gje3xUsWz2Z-WaK9wYx4%3DhCY1zN%3D7%2BBQ%40mail.gmail.com) 5.0, which is an open source implementation of the popular TPC/C OLTP database benchmark.\n\nIn terms of stand-alone performance, openGauss with ShardingSphere broke the limit of multi-core CPU: two-way 128-Core Huawei Kunpeng reached 1.5 million tpmC, and the memory-optimized table (MOT) engine reached 3.5 million tpmC.\n\nThese are great results, but we’re not done. We’ll never stop pushing the boundaries for better database performance — especially in today’s Big Data scenarios and their thirst for top notch database performance.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/tz0oyppd6vh7l1466ug5.png)\n\nIn this case, the openGauss team used 7 machines to run BenchmarkSQL adapted to [ShardingSphere-JDBC](https://shardingsphere.apache.org/document/current/en/overview/#shardingsphere-jdbc), connected 8 openGauss databases, and deployed [ShardingSphere-Proxy](https://shardingsphere.apache.org/document/current/en/overview/#shardingsphere-jdbc) for data initialization, consistency verification, and other maintenance operations.\n\nThanks to its database sharding capability, ShardingSphere enabled a total of 8,000 bins of data (over 800 GB) to be distributed across 8 openGauss nodes. Following over 1 hour of test, not only sharding was perfect but the average results also reached over 10 million tpmC, which is the best industry performance at this scale.\n\n\n## ShardingSphere & openGauss: Building an Ecosystem Cooperation\n\nThe Apache ShardingSphere community has been working closely with the openGauss community since 2021.\n\nFaced with the diversification of business scenarios and data volume expansion, the traditional solution that centrally stores data to a single node has since become unable to meet needs in terms of performance, availability, and affordable operation cost.\n\nDatabase sharding can solve problems of performance, availability, as well as single-point backup and recovery of stand-alone databases — but it also makes distributed architecture more complex.\n\nAs the proponent of the Database Plus concept, Apache ShardingSphere aims to build a criterion and ecosystem above heterogeneous databases and enhance the ecosystem with sharding, elastic scaling, encryption features & more. Placed above databases, ShardingSphere focuses on the collaborative way of databases to make reasonable and full use of database compute and storage capabilities.\n\nCurrently Apache ShardingSphere has a microkernel plus plugin-oriented architecture model, and on this basis, it continues to improve the capabilities of its kernel and functions to provide increasingly flexible solutions.\n\nThanks to the design concept of its pluggable architecture, ShardingSphere can support openGauss without additional changes and only needs to increase implementations of the corresponding openGauss database based on the SPI extension points provided by each ShardingSphere module .\n\nOur two communities have collaborated to create a distributed database solution suitable for highly-concurrent Online Transaction Processing (OLTP) scenarios by combining the powerful standalone performance of openGauss with the distributed capabilities provided by the Apache ShardingSphere ecosystem.\n\n\n## Building an openGauss-based Distributed Database Solution with ShardingSphere\nApache ShardingSphere includes many features such as database sharding, read/write splitting, data encryption, and shadow database. The features can be used independently or in combination.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/jq8uhm2w25yk8r3n8xvh.png)\n\nCurrently, ShardingSphere provides users with two access methods, namely ShardingSphere-JDBC and ShardingSphere-Proxy.\n\nShardingSphere-JDBC can easily and transparently perform operations such as sharding and read/write splitting on databases while meeting high concurrency and low latency needs.\n\nShardingSphere-Proxy is deployed to add some database capabilities and operations at the proxy level, enabling users to operate ShardingSphere as if it was a native database for a better user experience.\n\nShardingSphere-JDBC and ShardingSphere-Proxy can be deployed together. We recommend using this mixed deployment in order to make the system user-friendly and perform better.\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/nla348ta7qgxuwa8ghb6.png)\n\nFrom the perspective of the openGauss system, Apache ShardingSphere can shard the database horizontally to greatly enhance compute and storage capabilities, as well as database performance.\n\nThis means it can effectively solve problems caused by increasing data volume in a single table and can be combined with business data flows to flexibly and smoothly scale out data nodes, intelligently split reads and writes, and implement automatic load balancing of distributed databases.\n\n\n## Conclusion\nApache ShardingSphere and openGauss can seek potential cooperation opportunities.\n\nConsidering the increasingly diversified applicaiton scenarios and increasing data volume, the requirements for database performance are at an all time high and will only continue to increase in the future.\n\nThe success of our two communities cooperation is just the beginning of ourtwo communities building a collaborative database ecosystem.\n\n💡 About openGauss\n\nopenGauss is an open source relational database management system. It has enterprise-grade features such as multi-core high performance, full-link security, and intelligent operation.\n\nIt integrates Huawei’s years of kernel development experience in the database field and makes adaptations and optimizations on architecture, transaction, storage engine, optimizer, and ARM architecture.\n\n💡 About TPC-C\n\nTransaction Processing Performance Council Benchmark C or TPC-C is a benchmark used to compare the performance of online transaction processing (OLTP) systems. It was released by Transaction Processing Performance Council (TPC) in 1992. The latest update is TPC-C v5.11 published in 2010.\n\nTPC-C involves a mix of five concurrent transactions of different types and complexity either executed online or queued for deferred execution. The database is comprised of nine types of tables with a wide range of record and population sizes.\n\nTPC-C is measured in transactions per minute (tpmC). While the benchmark portrays the activity of a wholesale supplier, TPC-C is not limited to the activity of any particular business segment, but, rather represents any industry that must manage, sell, or distribute a product or service.\n\n\nApache ShardingSphere Project Links:\n\n[ShardingSphere Github](https://github.com/apache/shardingsphere/issues?page=1&q=is%3Aopen+is%3Aissue+label%3A%22project%3A+OpenForce+2022%22)\n\n[ShardingSphere Twitter](https://twitter.com/ShardingSphere)\n\n[ShardingSphere Slack](https://join.slack.com/t/apacheshardingsphere/shared_invite/zt-sbdde7ie-SjDqo9~I4rYcR18bq0SYTg)\n\n[Contributor Guide](https://shardingsphere.apache.org/community/cn/contribute/)","date":"2022-04-21","author":"SphereEx","excerpt":"ShardingSphere collaborated with openGauss to create a distributed database solution suitable for highly-concurrent Online Transaction Processing (OLTP) scenarios.","metaDescription":"openGauss is an open source relational database management system that is released with the Mulan PSL v2. with the kernel built on Huawei’s years of experience in the database field and continuously provides competitive features tailored to enterprise-grade scenarios. In addition, openGauss is an open source database platform that encourages community contribution and collaboration.","createdAt":"2022-09-26T08:12:02.238Z","updatedAt":"2022-09-27T07:00:09.560Z","publishedAt":"2022-09-27T07:00:09.551Z","locale":"en","caseStudiesType":{"data":{"id":10,"attributes":{"name":"Software Development","createdAt":"2022-09-26T10:34:53.523Z","updatedAt":"2022-09-27T06:59:43.907Z","publishedAt":"2022-09-27T06:59:43.905Z","locale":"en"}}},"cover":{"data":{"id":348,"attributes":{"name":"openGauss2.png","alternativeText":"openGauss2.png","caption":"openGauss2.png","width":720,"height":192,"formats":{"thumbnail":{"name":"thumbnail_openGauss2.png","hash":"thumbnail_open_Gauss2_da22098be0","ext":".png","mime":"image/png","width":245,"height":65,"size":9.07,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_open_Gauss2_da22098be0.png"},"small":{"name":"small_openGauss2.png","hash":"small_open_Gauss2_da22098be0","ext":".png","mime":"image/png","width":500,"height":133,"size":26.12,"path":null,"url":"https://official-media.sphere-ex.com/small_open_Gauss2_da22098be0.png"}},"hash":"open_Gauss2_da22098be0","ext":".png","mime":"image/png","size":34.31,"url":"https://official-media.sphere-ex.com/open_Gauss2_da22098be0.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:11:48.341Z","updatedAt":"2022-09-26T08:11:48.341Z"}}},"metaLogo":{"data":{"id":361,"attributes":{"name":"openGauss logo.png","alternativeText":"openGauss logo.png","caption":"openGauss logo.png","width":275,"height":183,"formats":{"thumbnail":{"name":"thumbnail_openGauss logo.png","hash":"thumbnail_open_Gauss_logo_7b11da9186","ext":".png","mime":"image/png","width":234,"height":156,"size":11.4,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_open_Gauss_logo_7b11da9186.png"}},"hash":"open_Gauss_logo_7b11da9186","ext":".png","mime":"image/png","size":3.07,"url":"https://official-media.sphere-ex.com/open_Gauss_logo_7b11da9186.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T09:11:52.021Z","updatedAt":"2022-09-26T09:11:52.021Z"}}},"localizations":{"data":[]}}},{"id":21,"attributes":{"feature":false,"title":"Apache ShardingSphere & CITIC Industrial Cloud","content":"_To further understand application scenarios, and enterprises’ needs, and improve dev teams’ understanding of Apache ShardingSphere, our community launched the “Enterprise Visits” series._\n\nOn February 28, the Apache ShardingSphere core development team visited CITIC Industrial Cloud Co., Ltd at the invitation of the company. During the meetup, Zhang Liang, Apache ShardingSphere PMC Chair and SphereEx Founder provided details on open source ecosystem building, and ShardingSphere production scenario-oriented solutions.\n\n[CITIC Group](https://www.citic.com/en/#Businesses) is China’s largest conglomerate. It is engaged in a wide variety of businesses, from property development to heavy industry to resources, though its primary focus is the financial sector, including banking and securities.\n\n[CITIC Industrial Cloud Co., Ltd](https://www.citictel-cpc.com/en-eu/product-category/cloud-computing), fully owned by CITIC Group, has developed an open service platform providing industrial internet applications and services in multiple fields such as IoT, smart cities, smart water services — with a desire to promote the digital transformation of traditional industries.\n\nNow, CITIC Industrial Cloud is exploring how to innovate with existing database systems, to support the innovative development of multiple industries.\n\nAs one of the most prominent open source middleware projects, with its ever-expanding ecosystem,[Apache ShardingSphere](https://shardingsphere.apache.org/) can meet some of CITIC Industrial Cloud’s technology needs in its development roadmap.\n\n\n\n## Ensuring Data Consistency in Data Migration\n\nEnterprises are challenged by ever expanding data related issues. Luckily, Scale-out can solve this problem.\n\nApache ShardingSphere allows online elastic scaling through its [Scaling tool ](https://shardingsphere.apache.org/document/current/en/features/scaling/)and provides multiple built-in data consistency verification algorithms, to ensure the consistency between data source and data on the client-side.\n\n[CRC32](https://crc32.online/) is used by default to strike a balance between speed and consistency; while the verification algorithm supports user-defined SPI.\n\nSince the time data migration takes is proportional to data volumes, to ensure data consistency, a downtime window is required to make the log level. Data will be verified through Scaling, at the business layer, and database layer.\n\nApache ShardingSphere can divide data migration into multiple sessions and execute them at the same time, merge modification operations of the same record, and execute a line command after configuration to stop write into the primary database, so the database is read-only.\n\nAdditionally, SQL execution is suspended during the read-only period since the data is globally consistent when write is disabled. Accordingly, the impact on system usability will be reduced when performing database switches.\n\n## JDBC and Proxy are Designed for Different Core Users\nApache ShardingSphere now supports access through JDBC and Proxy together with Mesh in the cloud (TODO). Users can choose the product that can best suit their needs to perform operations such as data sharding, read/write splitting, and data migration on original clusters.\n\n[ShardingSphere-JDBC](https://shardingsphere.apache.org/document/current/en/overview/#shardingsphere-jdbc) is designed for Java developers. It’s positioned as a lightweight Java framework, or as an enhanced JDBC driver with higher performance. If the performance loss is between 2% and 7%, and you want to optimize the performance, ShardingSphere-JDBC can help reduce the loss to less than 0.1%. Developers can directly access databases through the JDBC client and the service is provided in `.jar`, which means no extra deployment or dependency is needed.\n\n[ShardingSphere-Proxy](https://shardingsphere.apache.org/document/current/en/overview/#shardingsphere-proxy) is a database management proxy designed for operation and maintenance personnel. Generally, when users need to access databases through Proxy, the network connection may lead to a 30% to 70% performance loss. Therefore, the performance of ShardingSphere-JDBC is much better than that of ShardingSphere-Proxy.\n\nHowever, the plugin-oriented architecture of ShardingSphere provides dozens of extension points based on SPIs. On the basis of these extension points, ShardingSphere by default implements functions such as data sharding, read/write splitting, data encryption, shadow database stress testing and high availability, and allows developers to extend these functions at will.\n\nShardingSphere-JDBC adopts a decentralized architecture, applicable to high-performance, lightweight OLTP Java applications.\n\nShardingSphere-Proxy provides static access and supports all languages, suitable for OLAP applications and sharding database management and operations.\n\nWhile JDBC can boost development efficiency, Proxy can deliver better O&M performance. Through [DistSQL](https://medium.com/nerd-for-tech/intro-to-distsql-an-open-source-more-powerful-sql-bada4099211), you can leverage Apache ShardingSphere as if you were using a database natively. This helps improve the development capabilities of Ops teams and enterprises’ data management capabilities.\n\nThanks to the combination of ShardingSphere-JDBC and ShardingSphere-Proxy, while adopting the same sharding strategy in one registry center, ShardingSphere can create an application system suitable for all scenarios. This database gateway-like model allows users to manage all underlying database clusters through Proxy and observe distributed cluster status through SQL, and it can, therefore, enable maintainers and architects to adjust the system architecture to the one that can perfectly meet their business requirements and facilitate business operations.\n\n## ShardingSphere Federated Queries\nSQL Federation queries are a query mode providing cross-database querying capabilities. Users can execute queries without storing the data in the same database.\n\nThe SQL Federation engine contains processes such as SQL Parser, SQL Binder, SQL Optimizer, Data Fetcher and Operator Calculator, suitable for dealing with co-related queries and subqueries cross multiple database instances. At the underlying layer, it uses [Calcite](https://calcite.apache.org/) to implement RBO (Rule Based Optimizer) and CBO (Cost Based Optimizer) based on relational algebra, and query the results through the optimal execution plan.\n\nAs concepts such as data warehouse and data lake are gaining popularity, the applications of ShardingSphere SQL Federation are multiple.\n\nIf the user needs to perform a Federation query in a relational database, it can be easily implemented by ShardingSphere. Although data lake deployment is rather complex, ShardingSphere allows federated computations for data inside and outside the data lake by docking with databases, achieving `LEFT OUTER JOIN`,` RIGHT OUTER JOIN`, complex aggregate queries, etc.\n\n## Apache ShardingSphere Project Links:\n[ShardingSphere Github](https://github.com/apache/shardingsphere/issues?page=1&q=is%3Aopen+is%3Aissue+label%3A%22project%3A+OpenForce+2022%22)\n\n[ShardingSphere Twitter](https://twitter.com/ShardingSphere)\n\n[ShardingSphere Slack](https://apacheshardingsphere.slack.com/ssb/redirect)\n[\nContributor Guide](https://shardingsphere.apache.org/community/cn/contribute/)\n\n## Author\n**Yacine Si Tayeb**\n\nSphereEx Head of International Operations\nApache ShardingSphere Contributor\nPassionate about technology and innovation, Yacine moved to Beijing to pursue his Ph.D. in Business Administration and fell in awe of the local startup and tech scene. His career path has so far been shaped by opportunities at the intersection of technology and business. Recently he took on a keen interest in the development of the ShardingSphere database middleware ecosystem and Open-Source community building.\n\n\n\n\n\n\n","date":"2022-03-29","author":"SphereEx","excerpt":"CITIC Group is China's largest conglomerate. Now, CITIC is exploring how to innovate with existing database systems together with Apache ShardingSphere.\n","metaDescription":"CITIC Limited is the conglomerate in China and an established global player, with businesses covering financial services, resources and energy, manufacturing, engineering contracting and real estate as well as others.","createdAt":"2022-09-26T07:30:47.927Z","updatedAt":"2022-09-27T07:00:13.601Z","publishedAt":"2022-09-27T07:00:13.595Z","locale":"en","caseStudiesType":{"data":{"id":1,"attributes":{"name":"Finance","createdAt":"2021-12-11T23:01:56.177Z","updatedAt":"2021-12-11T23:01:58.635Z","publishedAt":"2021-12-11T23:01:58.567Z","locale":"en"}}},"cover":{"data":{"id":341,"attributes":{"name":"CITIC.png","alternativeText":"CITIC.png","caption":"CITIC.png","width":720,"height":389,"formats":{"thumbnail":{"name":"thumbnail_CITIC.png","hash":"thumbnail_CITIC_6be834d667","ext":".png","mime":"image/png","width":245,"height":132,"size":59.75,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_CITIC_6be834d667.png"},"small":{"name":"small_CITIC.png","hash":"small_CITIC_6be834d667","ext":".png","mime":"image/png","width":500,"height":270,"size":216.53,"path":null,"url":"https://official-media.sphere-ex.com/small_CITIC_6be834d667.png"}},"hash":"CITIC_6be834d667","ext":".png","mime":"image/png","size":295.62,"url":"https://official-media.sphere-ex.com/CITIC_6be834d667.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T07:31:53.040Z","updatedAt":"2022-09-26T07:31:53.040Z"}}},"metaLogo":{"data":{"id":355,"attributes":{"name":"CITIC logo.jpg","alternativeText":"CITIC logo.jpg","caption":"CITIC logo.jpg","width":960,"height":540,"formats":{"thumbnail":{"name":"thumbnail_CITIC logo.jpg","hash":"thumbnail_CITIC_logo_69bc3ea16b","ext":".jpg","mime":"image/jpeg","width":245,"height":138,"size":4.77,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_CITIC_logo_69bc3ea16b.jpg"},"medium":{"name":"medium_CITIC logo.jpg","hash":"medium_CITIC_logo_69bc3ea16b","ext":".jpg","mime":"image/jpeg","width":750,"height":422,"size":18.97,"path":null,"url":"https://official-media.sphere-ex.com/medium_CITIC_logo_69bc3ea16b.jpg"},"small":{"name":"small_CITIC logo.jpg","hash":"small_CITIC_logo_69bc3ea16b","ext":".jpg","mime":"image/jpeg","width":500,"height":281,"size":11.89,"path":null,"url":"https://official-media.sphere-ex.com/small_CITIC_logo_69bc3ea16b.jpg"}},"hash":"CITIC_logo_69bc3ea16b","ext":".jpg","mime":"image/jpeg","size":20.09,"url":"https://official-media.sphere-ex.com/CITIC_logo_69bc3ea16b.jpg","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:48:30.170Z","updatedAt":"2022-09-26T08:48:30.170Z"}}},"localizations":{"data":[]}}},{"id":14,"attributes":{"feature":false,"title":"Asia’s E-Commerce Giant Dangdang Increases Order Processing Speed by 30% — Saves Over Ten Million in Technology Budget with  Apache ShardingSphere","content":">Apache ShardingSphere is an easy-to-use and stable product, making Dangdang’s warehouse management system (WMS) even more powerful. Integrated with WMS, ShardingSphere plays a vital role in reforming the supply chain system.\n>-- Li Yong, Head of WMS Technology, Dangdang\n\nFollowing Apache ShardingSphere 5.0.0 GA release in November 2021, the 5.1.0 version was released last month. Having gone through over two years of polishing, ShardingSphere’s plugin-oriented ecosystem is beginning to take shape, and the project embarks on the evolution from a simple data sharding middleware to a mature distributed database ecosystem driven by the concept of Database Plus.\n\nDangdang, established at the end of 1999, has become a leading e-commerce platform selling books of any kind, and by integrating new Internet technologies with the traditional book industry. Dangdang was founded during the surge in China’s Internet industry in the early 2000s.\n\nLater, the e-commerce industry became exremely competitive in the country, and the market saturated. Facing fierce market competition, e-commerce platforms had to adapt to remain competitive.\n\nIn response, Dangdang not only adjusted its business strategies and management approaches, but also upgraded its technology architecture. Dangdang didn’t have its warehouse management and transportation system at that time. However, with growing business volume and technological capabilities, Dangdang needed to rebuild its warehouse management system and transportation management system (TMS) to better satisfy its business needs. For instance, in terms of hardware, it replaced mini-computer with x86, while its old centralized system was transformed to a distributed system with more flexibility.\n\nOne of the biggest challenges was massive warehousing data storage. The engineers wanted to adopt the data sharding technology that was often chosen by other big Internet companies. Disappointingly, they failed to find a mature and versatile open source database middleware in the marketplace, and therefore,started to develop a new data sharding product. That’s the origin of Sharding-JDBC. The product was created to bring more possibilities to data services.\n\nDangdang released its new WMS five years ago, which meant that it completed its intelligent warehousing transformation. Since then, with Apache ShardingSphere, the WMS has enabled Dangdang to hold large online shopping events every year such as Dangdang’s April Reading Festival, Double-Eleven Online Shopping Festival (aka Singles Day), and Mid-Year Shopping Festival, and to manage over a dozen of smart warehouses.\n\n## Business Challenges\n\nWhen Dangdang adopted a third-party WMS, the database in use was Oracle based on IBM minicomputers.\n\nHowever, considering the increasing business volume and warehouse order requests, especially during the online shopping festivals, the traditional centralized database architecture of the old WMS became inadequate because the computing and storage capabilities were limited. Additionally, the scale-up solution couldn’t support the system during online shopping festivals, and therefore the developers must do scale-up and adjust the business layer several times to alleviate the storage and computing limits and avoid production risks.\n\n- **Limited computing and storage capabilities**\n\nA centralized architecture is less scalable, making database computing and storage capabilities become the bottleneck.\n\n- **Expensive development and maintenance cost**\n\nBecause of poor scalability, developers have to make concessions and scale-up, which increases system development and maintenance costs.\n\n- **Exclusiveness**\n\nIf the architecture is not open enough, the system is less flexible, with fewer functions, and difficult to be transformed. The current architecture makes it difficult to quickly adopt new business services such as cloud native ones, SQL audit, data encryption, and distributed governance.\n\n## The Solution\n\nBased on the situation described above, Dangdang’s tech team proposed a warehouse management system solution: in terms of hardware, IBM minicomputer would be replaced with all-purpose x86, and MySQL would replace Oracle.\n\nHowever, at that time, there wasn’t a versatile and mature enough open source database middleware living up to Dangdang’s expectations, so they created one and named it Sharding-JDBC.\n\nShardingSphere-JDBC is positioned as a lightweight Java framework that provides additional services at the Java Database Connectivity (JDBC) layer. It is lightweight, efficient, easy to use, and compatible.\n\nWith ShardingSphere providing services in the form of the .jar files package, users can connect the client directly to the database without additional deployment and dependencies. It can be seen as an enhanced JDBC driver, fully compatible with JDBC and all ORM frameworks.\n\n- Compatible with JDBC and any JDBC-based ORM framework such as JPA, Hibernate, Mybatis, Spring JDBC Template.\n\n- Supports all third-party database connection pools such as DBCP, C3P0, BoneCP, HikariCP, etc.\n\n- Supports all databases implementing JDBC standards. Currently, ShardingSphere-JDBC supports MySQL, PostgreSQL, Oracle, SQL Server, and any database that can be accessed via JDBC.\n\n![1.jpeg](https://official-media.sphere-ex.com/1_3d03cbca69.jpeg)\nThe ShardingSphere-JDBC Topography\n\nCurrently, Apache ShardingSphere consists of three products, i.e. JDBC, Proxy, and Sidecar (TODO). ShardingSphere-JDBC and ShardingSphere-Proxy can be deployed independently or together.\n\nIt is ShardingSphere-JDBC that is used in Dangdang’s warehouse management system.\n\n\n>So how is ShardingSphere-JDBC exactly utilized?\n\nIn the warehouse management system, each warehouse positioned in a physical city is referred to as a unit with its corresponding business system and databases. Each warehouse has three sets of MySQL primary-secondary clusters to load the warehousing data of the designated city. So far, Dangdang has more than ten self-built warehouses all over China, mostly in cities where customers place a large number of orders. This self-built warehouse model is flexible for warehouse management and helps reduce storage costs in the long run.\n\nIn terms of architecture, the WMS uses ShardingSphere-JDBC to do database sharding according to their business types, and each cluster stores specified business data. The three MySQL clusters of a single warehouse are divided into three types as follows:\n\n-** Basic:** stores user, area, and menu data.\n- **Business:** stores order and package data.\n- **Inventory:** stores stock and working data.\n\nBefore the release, the system was initialized based on the basic data of a warehouse, such as storage locations.\n\nNext, by deploying the distributed database middleware, Dangdang successfully solved a series of problems such as limited storage and computing capabilities, high costs, and lack of flexibility.\n\n![1.jpeg](https://official-media.sphere-ex.com/1_b7199ae49f.jpeg)\nTopography of Dangdang’s Warehouse Management System (of a Single Warehouse)\n\n## User Advantages\n\nApache ShardingSphere played a significant role in helping Dangdang develop its WMS. There are five main benefits:\n\n- **Extraordinary performance**\n\nThe ShardingSphere-JDBC lightweight framework makes its performance close to a native JDBC. Apart from its great database sharding capability, it helps database performance be taken to the extreme, which allows the WMS to work at full capacity.\n\n- **Keep the system stable**\n\nWMS has been functioning well since its release in 2016.\n\n- **Low risk & zero invasion**\n\nThe underlying system of Dangdang has been evolving since 2000. Thanks to its zero-intrusion nature, Apache ShardingSphere can be compatible with others with small modifications to meet Dangdang’s business requirements.\n\n- **Allow developers to focus on the business side**\n\nThe developer team does not need to worry about sharding any more and can concentrate on developing the system to meet business needs.\n\n- **Cost effective and efficient**\n\nSince ShardingSphere is known for its high compatibility, to satisfy increasing business needs, developers don’t have to reconstruct or upgrade the system, minimizing the migration cost.\n\nWarehousing order processing speed is increased by 30% and accordingly, tens of millions of manpower costs are reduced due to the smart warehouses and the auto storage location matching technology.\n\n**Apache ShardingSphere’s Roadmap**\n\nSome said that ShardingSphere is a product created by Dangdang. To be precise, ShardingSphere was derived from the company and it also donated ShardingSphere to the Apache Software Foundation (ASF) on November 10, 2018.\n\nAfter 17-months in the ASF incubator, Apache ShardingSphere successfully graduated on April 15, 2020, as a Top-Level Apache Project.\n\nRecently, to celebrate the third anniversary of ShardingSphere entering Apache Software Foundation(ASF), the community released ShardingSphere 5.0.0. Below is a brief review of Apache ShardingSphere.\n\n- In 2014, Dangdang introduced a centralized development framework targeting at its e-commerce platform called dd-frame. It was created to unify the development framework, standardize its technical components, and achieve efficient cross-team communication by separating business code from technical code. In this way, engineers can devote all their efforts to the business side. The relational database module named dd-rdb in the framework was developed to handle data access and implement the data sharding function. It was the precursor of Sharding-JDBC, as well as a major part of dd-frame 2.x.\n\n- In 2015, Dangdang decided to rebuild its WMS and TMS. As it needed a data sharding plan, the team launched the project in September. In December, 2015, Sharding-JDBC 1.0.0 was released and used within Dangdang.\n\n- In early 2016, Sharding-JDBC was separated from dd-rdb and became open source. The product is an enhanced JDBC driver providing service in .jar files.\n\n- At the end of 2017, Version 2.0.0 was released with the new data governance function.\n\n- In 2018, ShardingSphere was enrolled into Apache Incubator. The release of Version 3.0.0 was a notable turnaround: Sharding-Proxy was released as an independent service. It supported heterogeneous languages, and the project was renamed from Sharding-JDBC to ShardingSphere. It’s in 2018 that the community decided to build the criteria and ecosystem above databases.\n\n- In 2019, Version 4.0.0 was released capable of supporting more database products.\n\n- In 2020, ShardingSphere graduated as a Top-Level Project of the ASF.\n\n- On November 10, 2021, Version 5.0.0 GA was released as a third-anniversity celebration with the whole Apache ShardingSphere community, and the distributed database industry.\n\n![1.jpeg](https://official-media.sphere-ex.com/1_83a06f3e37.jpeg)\nApache ShardingSphere—Roadmap\n\nSince Version 5.0.0, Apache ShardingSphere has embarked on its new journey: with the plugin oriented architect at its core, it evloved from a data sharding application to a comprehensive and enhanced data governance tool applicable to various complex application scenarios. Concurrently, Apache ShardingSphere also has more features, and big data solutions.\n\n## Conclusion\n\nDigitization motivated Dangdang to achieve high-quality development and fulfill its mission. ShardingSphere is glad to support Dangdang’s WMS with its cutting-edging data services.\n\nHaving gone through two years‘ development, Apache ShardingSphere 5.0.0 GA has been released. The pluggable ecosystem marks an evolution from a data sharding middleware tool to a pioneer in the industry following the “Database Plus” concept.","date":"2022-03-11","author":"SphereEx","excerpt":"Apache ShardingSphere is easy-to-use and stable, making Dangdang’s warehouse management system (WMS) even more powerful.","metaDescription":"Dangdang is a B2C-based e-commerce company in China offering a large selection of products both online and offline.","createdAt":"2022-04-29T10:52:10.330Z","updatedAt":"2022-09-28T02:42:43.896Z","publishedAt":"2022-04-29T10:52:12.009Z","locale":"en","caseStudiesType":{"data":{"id":9,"attributes":{"name":"Internet","createdAt":"2022-09-26T10:34:03.086Z","updatedAt":"2022-09-27T06:59:34.838Z","publishedAt":"2022-09-27T06:59:34.836Z","locale":"en"}}},"cover":{"data":{"id":340,"attributes":{"name":"dangdang1.jpeg","alternativeText":"dangdang1.jpeg","caption":"dangdang1.jpeg","width":720,"height":466,"formats":{"thumbnail":{"name":"thumbnail_dangdang1.jpeg","hash":"thumbnail_dangdang1_27d7d16721","ext":".jpeg","mime":"image/jpeg","width":241,"height":156,"size":25.62,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_dangdang1_27d7d16721.jpeg"},"small":{"name":"small_dangdang1.jpeg","hash":"small_dangdang1_27d7d16721","ext":".jpeg","mime":"image/jpeg","width":500,"height":324,"size":91.72,"path":null,"url":"https://official-media.sphere-ex.com/small_dangdang1_27d7d16721.jpeg"}},"hash":"dangdang1_27d7d16721","ext":".jpeg","mime":"image/jpeg","size":153.14,"url":"https://official-media.sphere-ex.com/dangdang1_27d7d16721.jpeg","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T07:12:49.980Z","updatedAt":"2022-09-26T07:12:49.980Z"}}},"metaLogo":{"data":{"id":351,"attributes":{"name":"dangdang logo.jpg","alternativeText":"dangdang logo.jpg","caption":"dangdang logo.jpg","width":500,"height":256,"formats":{"thumbnail":{"name":"thumbnail_dangdang logo.jpg","hash":"thumbnail_dangdang_logo_539787f05a","ext":".jpg","mime":"image/jpeg","width":245,"height":125,"size":5.42,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/thumbnail_dangdang_logo_539787f05a.jpg"}},"hash":"dangdang_logo_539787f05a","ext":".jpg","mime":"image/jpeg","size":12.11,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/dangdang_logo_539787f05a.jpg","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:37:49.195Z","updatedAt":"2022-10-12T05:57:41.631Z"}}},"localizations":{"data":[{"id":13,"attributes":{"feature":true,"title":"ShardingSphere 助力当当 WMS：订单效率提升 30%、节约成本上千万","content":">Apache ShardingSphere 简单易用且成熟稳定，当当自研 WMS 如虎添翼，在供应链系统改革中起到了至关重要的作用，是我们的杀手锏。\n——李勇，当当 WMS 技术负责人\n\n>Apache ShardingSphere 5.0.0 GA 已于 2021 年 11 月发布，5.1.0 版本也于上周发布。历经两年的迭代与打磨，可插拔生态趋近成熟，标志着 Apache ShardingSphere 从分库分表中间件蜕变为以 Database Plus 为理念的分布式数据库生态系统。\n\n近年来，随着“建设社会主义文化强国”政策的持续落实，全民阅读在移动互联快速发展中得到大力倡导，阅读给当代人提供了丰富的精神食粮。当当成立于 1999 年底，作为新技术与传统行业的结合体，当当融合了互联网新技术和传统图书行业，二十余年深耕图书领域助力全民阅读。\n\n当当在刚成立的几年，也是中国互联网腾飞的阶段，电商逐渐多元化发展的趋势迎来了行业野蛮生长期，各电商平台的发展如逆水行舟，不进则退。当当对战略和管理等方面进行调整的同时，也对技术架构层进行了硬核升级。\n\n当当成立之初的仓储物流系统并非自研，随着业务体量、技术实力等多方面的变化，为了满足业务快速发展需要，当当决定重构 WMS（Warehouse Management System）和 TMS（Transportation Management System）。硬件层使用 x86 取代小机，数据层由传统集中式转型为更灵活的分布式架构。如何解决海量仓储货品信息问题，是重构待解决的核心问题之一。互联网技术栈比较常用的分库分表被率先尝试使用，但由于当时开源数据库中间件的成熟度及功能方面不满足预期，当当决定自研分库分表产品，Sharding-JDBC 就这样应运而生，为“连接”带来更多可能。\n\n**自研 WMS 的上线完成了仓储层智能改革，五年、十余仓，Apache ShardingSphere 同 WMS 完美支撑了数个“读书节”、“618”和“11.11”等大型促销活动。**\n\n\n## 业务挑战\n\n当当在自研 WMS 之前使用的是第三方仓储管理系统，其数据库是基于 IBM 小机的 Oracle，结合当当业务体量以及电商仓储物流在促销期间密集请求的业务特点，传统集中式数据库架构给 WMS 带来了极大的挑战。\n\n在业务快速发展的冲击下，数据库计算和存储能力的上限越发明显，Scale-up 方案难以满足超级促销期间的业务需求，技术团队需要对业务层多次做拆分调整来缓解“存算”瓶颈，产能存在受限的风险。\n\n- 计算、存储能力双受限\n\n传统集中式架构扩展能力相对较差，数据库的计算和存储能力成为明显瓶颈；\n\n- 业务开发维护成本高昂\n\n在扩展能力相对差的基础上，业务侧需要不断的去让步做拆分，开发和维护成本持续攀升；\n\n- 系统架构开放度不足\n\n系统架构开放度不足将会使系统面临灵活度欠佳、功能缺失以及转型难度高等问题，基于现有架构难以快速响应新业务的开展及相关政策的调整，如云原生、SQL 审计、数据脱敏以及分布式治理等多方面需求。\n\n\n## 解决方案\n\n当当技术部结合现状及未来发展等多维度的调研，最终确定了自研 WMS 的方案。在硬件方面全面采用通用的 x86 设备替换 IBM 小型机，数据库层面引入开源数据库 MySQL 来替换 Oracle。当时已有多个开源数据库中间件，但成熟度和功能方面与预期不符，因此选择了自研数据库中间件，Sharding-JDBC 闪亮登场。\n\nShardingSphere-JDBC 定位为轻量级 Java 框架，在 Java 的 JDBC 层提供额外服务。具备轻量、高效、易用以及高兼容等特点。它使用客户端直连数据库，以 jar 包形式提供服务，无需额外部署和依赖，可理解为增强版的 JDBC 驱动，完全兼容 JDBC 和各种 ORM 框架。\n\n- 适用于任何基于 JDBC 的 ORM 框架，如：JPA、Hibernate、Mybatis、Spring JDBC Template 或直接使用 JDBC；     \n\n- 支持任何第三方的数据库连接池，如：DBCP、C3P0、BoneCP、HikariCP 等；\n\n- 支持任意实现 JDBC 规范的数据库，目前支持 MySQL、PostgreSQL、Oracle、SQL Server 以及任何可使用 JDBC 访问的数据库。\n\n![当当1.png](https://official-media.sphere-ex.com/1_16c3bbe14b.png)\n\nShardingSphere-JDBC 方案\n\n目前 Apache ShardingSphere 由 JDBC、Proxy 和 Sidecar（规划中）这 3 款既能够独立部署，又支持混合部署配合使用的产品组成，在当当自研 WMS 中使用的是 ShardingSphere-JDBC。\n\n那么 ShardingSphere-JDBC 在 WMS 中具体是如何应用的呢？\n\n在 WMS 系统中，城市的“仓”就是仓储系统的管理单位，每个仓有对应的业务系统和数据库，单个仓使用 3 套 MySQL 主从集群承载指定城市仓库的数据。目前，当当在全国各地设有十余个自建的仓库，主要分布在订单量较大的城市，这种自建仓库模式在具备灵活管理等特性的同时，也可以降低企业的长期仓储成本。\n\n在架构规划上，WMS 通过 ShardingSphere-JDBC 按业务类型进行分库， 每套集群负责存储指定的业务数据，单个仓的 3 套 MySQL 集群分工如下：\n\n- 基础库集群：User，Area，Menu；\n\n- 业务库集群：Order，Package；\n\n- 库存库集群：Stock，Working。\n\n上线前，系统会根据仓库的基础数据进行初始化，如货位等信息，这里不做深入讨论。\n\n分布式数据库中间件的架构解决了 WMS 存算受限、开发维护成本高以及系统灵活度不足的问题，以成熟的底座和熟悉的技术栈去承载业务，架构相对透明化，兼容度良好且没有入侵性。\n\n![当当2.png](https://official-media.sphere-ex.com/2_783843d774.png)\n\n当当 WMS 系统拓扑图（单仓）\n\n\n## 用户收益\n\n在当当自研 WMS 的过程中发挥了非常重要的作用，带来了五方面的收益如下：\n\n1、性能极致\n\nShardingSphere-JDBC 轻量级的框架使其性能接近原生 JDBC，提供分库能力的同时性能可发挥极致，WMS 产能不再有受限的风险；\n\n2、系统稳定\n\n自 2016 年 WMS 上线至今，数年来系统一直稳定运行；\n\n3、低风险、零侵入\n\n当当底层系统为 2000 年演进而来，Apache ShardingSphere 凭借自身零侵入性的特点，用很小的改动就即完成了兼容适配，满足了业务需求；\n\n4、专注业务开发\n\n研发团队无需再关心业务拆分的工作，可将更多的精力聚焦于业务需求；\n\n5、降本增效\n\n高兼容的特点可以在无需大量的重构和改造前提下满足业务增长的需求，改造迁移成本低；     \n\n通过仓店智能化、入库商品自动匹配最优货位的设计，仓店订单效率提升了 30%，人力成本节约上千万。\n\n\n## Apache ShardingSphere 的前世今生\n\n有人说 ShardingSphere 是当当的产品，其实更准确的说：ShardingSphere 起源于当当，在 2018 年 11 月 10 日捐赠给 Apache，经 17 个月的孵化于 2020 年 4 月 15 毕业，是 2020 年度第一个从 Apache 孵化器毕业的顶级项目。\n\n前段时间，在 ShardingSphere 进入 Apache 基金会三周年的纪念日（2021/11/10）里，社区发布了期待已久的 5.0 正式版，借着这个机会，一起回顾 Apache ShardingSphere 的前世今生。\n\n- 2014 年，当当针对电商平台推出了统一开发框架 dd-frame，目的是通过分离业务和技术代码，实现开发框架统一化、技术组件标准化以及跨组沟通高效化，使研发人员的精力最大化的聚焦在业务开发上。开发框架中名为 dd-rdb 的关系型数据库模块专门用于负责数据访问，可实现简单的分库分表功能，它就是 Sharding-JDBC 的前身，也是 dd-frame 2.x 中重点规划的部分。     \n \n- 2015 年，当当决定重构 WMS 和 TMS，设计上需要引用分库分表方案，天时地利人和，当当架构部在 9 月正式启动了项目，同年 12 月份 Sharding-JDBC 1.0.0 在当当内部发布，正式开启了 ShardingSphere 的征途。\n \n- 2016 年初，Sharding-JDBC 从 dd-rdb 中剥离，向社区开源，产品形态是增强版 JDBC 驱动，以 jar 包形式提供服务。\n \n- 2017 年底，发布 2.0.0 版本，打开了数据库治理功能。\n\n- 2018 年，进入 Apache 软件基金会。3.0.0 版本是一个华丽的转身，独立服务形态的 Sharding-Proxy 面世，实现了异构开发语言的需求，同时项目名称从 Sharding-JDBC 更名为 ShardingSphere，迈出了构建数据库上层标准和生态的第一步。     \n \n- 2019 年，发布 4.0.0 版本，可支持更多的数据库产品。\n \n- 2020 年，毕业成为 Apache 软件基金会的顶级项目。    \n\n- 2021 年 11 月 10 日，发布 5.0.0 GA 版，作为加入 Apache 软件基金会三周年纪念日的礼物，呈现给社区及整个分布式数据库和安全生态领域。     \n\n![当当3.png](https://official-media.sphere-ex.com/3_ae2b522cd5.png)\n Apache ShardingSphere - RoadMap\n\n从 5.0.0 版本开始，Apache ShardingSphere 正式开启了以可插拔架构为核心的全新阶段，实现了由过去数据分片单一应用场景到现在复杂应用环境下综合数据治理的身份转变与能力提升。在新版本中，Apache ShardingSphere 具备了更多新特性，并拥有了更加成熟的大数据解决方案。\n\n首先，基于可插拔架构对内核进行了全面改造，内核中的各个功能可以任意组合并叠加使用。其次，为了提升 SQL 分布式查询能力，5.0.0 GA 版打造了全新的 Federation 执行引擎，来满足用户复杂的业务场景。此外，5.0.0 GA 版在内核功能 API 层面也进行了大量优化，旨在降低用户使用这些功能的成本。\n\n## 写在结尾\n\n全链条数字化驱动着当当高质量发展，让全民阅读触手可及。在必不可少的自研 WMS 一环中，ShardingSphere 在数据层提供了强力支撑。\n\nApache ShardingSphere 历经两年的迭代与打磨，5.0.0 GA 版本已面世。趋近成熟的可插拔生态，标志着 Apache ShardingSphere 从分库分表中间件工具蜕变为以 Database Plus 理念为新航标。重塑分布式可插拔体系，助力整个数据库行业带来新风向和更有价值的解决方案。\n\n更多内容欢迎点击阅读原文链接访问中文社区 OpenSEC（Open SphereEx Community），一起畅聊开源。\n\n","date":"2022-04-29","author":"SphereEx","excerpt":"历经两年的迭代与打磨，可插拔生态趋近成熟，标志着 Apache ShardingSphere 从分库分表中间件蜕变为以 Database Plus 为理念的分布式数据库生态系统。","metaDescription":null,"createdAt":"2022-04-29T08:49:24.970Z","updatedAt":"2022-11-02T05:36:20.351Z","publishedAt":"2022-04-29T08:49:28.304Z","locale":"zh"}}]}}},{"id":11,"attributes":{"feature":false,"title":"The SphereEx-Console Enterprise Grade Data Governance Platform is Officially Released","content":">On January 27th, SphereEx officially launched SphereEx-Console, a data monitoring and management platform. It provides a one-stop solution incorporating Apache ShardingSphere and SphereEx as the core, a powerful user interface to manage data clusters, and a platform to simplify user configuration.\n>\n>By simplifying operation & deployment processes, reducing entry barriers and data management challenges, SphereEx-Console improves enterprise operations and maintenance efficiency. \n\n\nExpanding data volumes and increasingly demanding agility requirements from front-end businesses have created significant challenges for enterprises in the finance, government and traditional manufacturing sectors. \n\nImproving underlying data architectures' adaptability, and upgrading it to a distributed ecosystem represent some of the new requirements - however, traditional centralized databases are inherently inadequate in terms of massive data carrying capacity. \n\nAs a result, distributed databases have emerged in recent years to meet this demand. Nevertheless, the complexity of distributed architectures poses great challenges to users. Simplifying users' experience has become a key database industry driver.\n\nAs a popular open source project, Apache ShardingSphere has successfully been implemented in multiple production scenarios, with an increasing number of enterprises adopting the Database Plus approach to transition from traditional relational databases to distributed databases. \n\nBased on Apache ShardingSphere a data platform on top of the existing databases is built, enabling distributed data capability to be deployed in the middle layer without requiring adjustments to the underlying data architecture. This approach allows enterprises to smoothly transition to distributed and cloud based platforms.\n\nThe SphereEx Enterprise Data Service Platform was created by leveraging ShardingSphere's open source kernel, and enhancing it with enterprise grade features. It provides improved data service capabilities, such as data sharding, data security etc.\n\n**SphereEx-Console is a powerful dashboard applied to the management and control of the SphereEx Enterprise Data Service Platform. It provides a one-stop solution with a user-friendly experience packaging resources, instances and plugins.**\n\nIt provides a visual approach to manage data clusters with Apache ShardingSphere and SphereEx as the core, while offering a platform to simplify user configuration, and ultimately improve enterprise operations & maintenance efficiency.\n\n## What is SphereEx-Console?\n\nThe SphereEx Enterprise Data Service Platform was created by leveraging ShardnginSphere's open source kernel, and enhancing it with enterprise grade features. It provides improved data service capabilities, such as data sharding, data security etc.\n\nSphereEx-Console is a powerful dashboard applied to the management and control of the SphereEx Enterprise Data Service Platform. It provides a one-stop solution with a user-friendly experience packaging multiple functions such as resource layer, instance layer and application layer.\n\nThe SphereEx-Console is used to manage the operation and maintenance of SphereEx Enterprise Data Service Platform. It supports operation, maintenance and development scenarios in Internet, finance and government industries.\n\nAs a SphereEx business data services cluster platform with ShardingSphere at its core, the SphereEx-Console visual data control platform provides a one-stop solution with a user friendly interface instead of complex code input, packaged with various features for the resource, instance and application layers. \n\n![Diagram Console EN.png](https://official-media.sphere-ex.com/Diagram_Console_EN_5f98720b3f.png)\n\n\n### SphereEx-Console Advantages\n\n**Easy to Use**\n\nIt improves the user experience and eliminates configuration errors. Users are not required to operate the SphereEx Enterprise Data Service Platform through configuration commands, greatly improving user experience. \n\n**Comprehensive**\n\nIt provides a unified solution for the management and control of the SphereEx Enterprise Data Service Platform - from basic resources to plugin capabilities.\n\n**Visual Monitoring**\n\nWith its data monitoring & management dashboard, SphereEx Enterprise Data Service Platform's clusters, instances and hosts monitoring data are visible online and in real time.\n\n### Industry Leading\n\nThe release of SphereEx-Console is an essential milestone for SphereEx's data management ecosystem. It improves and evolves Apache ShardingSphere in areas such as management difficulty and user interface. \n\nThe user interface greatly reduces enterprise operations & maintenance teams' learning costs, while the unified data management platform greatly improves operations and maintenance efficiency.\n\nAs a business founded the open source project Apache ShardingSphere's core team, SphereEx continues to enhance data governance as well as management & control capabilities. We are committed to providing users with a comprehensive Apache ShardingSphere application ecosystem and SphereEx enterprise grade capabilities.\n\n[Click here for more details!](https://sphere-ex.com/en/products/console/)","date":"2022-01-26","author":"SphereEx","excerpt":"Enhancing Enterprise Data Governance Capabilities","metaDescription":null,"createdAt":"2022-03-11T09:38:54.812Z","updatedAt":"2022-09-27T08:42:51.504Z","publishedAt":"2022-03-11T09:59:17.395Z","locale":"en","caseStudiesType":{"data":null},"cover":{"data":{"id":178,"attributes":{"name":"SphereEx-Console.png","alternativeText":"SphereEx-Console.png","caption":"SphereEx-Console.png","width":3751,"height":1596,"formats":{"thumbnail":{"name":"thumbnail_SphereEx-Console.png","hash":"thumbnail_Sphere_Ex_Console_46205ff4d3","ext":".png","mime":"image/png","width":245,"height":104,"size":26.2,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_Sphere_Ex_Console_46205ff4d3.png"},"large":{"name":"large_SphereEx-Console.png","hash":"large_Sphere_Ex_Console_46205ff4d3","ext":".png","mime":"image/png","width":1000,"height":425,"size":183.98,"path":null,"url":"https://official-media.sphere-ex.com/large_Sphere_Ex_Console_46205ff4d3.png"},"medium":{"name":"medium_SphereEx-Console.png","hash":"medium_Sphere_Ex_Console_46205ff4d3","ext":".png","mime":"image/png","width":750,"height":319,"size":129.65,"path":null,"url":"https://official-media.sphere-ex.com/medium_Sphere_Ex_Console_46205ff4d3.png"},"small":{"name":"small_SphereEx-Console.png","hash":"small_Sphere_Ex_Console_46205ff4d3","ext":".png","mime":"image/png","width":500,"height":213,"size":72.9,"path":null,"url":"https://official-media.sphere-ex.com/small_Sphere_Ex_Console_46205ff4d3.png"}},"hash":"Sphere_Ex_Console_46205ff4d3","ext":".png","mime":"image/png","size":345.31,"url":"https://official-media.sphere-ex.com/Sphere_Ex_Console_46205ff4d3.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-03-11T09:32:09.548Z","updatedAt":"2022-03-11T09:32:09.548Z"}}},"metaLogo":{"data":null},"localizations":{"data":[{"id":12,"attributes":{"feature":true,"title":"增强企业数据服务及管控能力｜SphereEx-Console 可视化数据管控平台正式发布","content":">**1 月 27 日，SphereEx 正式推出可视化数据服务管控平台：SphereEx-Console。**\n>\n>作为一款以管理 ShardingSphere 为核心的 SphereEx 商业数据服务集群平台，SphereEx-Console 以平台的形式简化用户配置，极大简化用户操作和部署流程，进而降低新手入门门槛和企业管控难度，进一步提升企业运维效率。\n\n随着数据体量增长以及前端业务敏捷性的要求，金融、政府、传统制造等领域均面临着提升底层数据架构适配性、面向分布式生态进行架构升级。但传统集中式数据库在海量数据承载能力方面存在先天性不足，于是近些年来分布式数据库不断涌现，很好地满足这一诉求。但我们也看到，分布式架构天然的复杂性，对使用者提出了更高的要求，如何在新架构下简化用户体验，便捷普惠地享受到新架构带来的优势，成为各家重点考虑的因素。\n\n作为近些年火热的开源项目，Apache ShardingSphere 的应用范围越来越广，越来越多的企业选择采用 “数据库 + ShardingSphere” 的方式来实现传统关系型数据库到分布式数据库的转型过渡。以 Apache ShardingSphere 为基础，在原有数据库之上构建一套数据中间平台，将分布式数据服务增强能力部署在中间层，而不需对底层数据架构做出调整，进而帮助企业以更加平滑的方式获得分布式以及云上迁移能力。\n\n因此，为进一步降低用户应用门槛，拓宽 Apache ShardingSphere 的使用范围，**SphereEx 在开源内核 Apache ShardingSphere 的基础上推出 SphereEx-Console 可视化数据管控平台。** 作为一款以管理 ShardingSphere 为核心的 SphereEx 商业数据服务集群平台，SphereEx-Console 以平台的形式简化用户配置，极大简化用户操作和部署流程，进而降低新手入门门槛和企业管控难度，进一步提升企业运维效率。\n\n## 什么是 SphereEx-Console\n\n**SphereEx 控制台主要用于对 SphereEx 数据平台的运维管控，在互联网、金融、政务行业均能够支持运维和开发人员常见的操作场景。**\n\n作为一款以管理 ShardingSphere 为核心的 SphereEx 商业数据服务集群平台，SphereEx-Console 可视化数据管控平台以简便易用的操作界面代替相对复杂代码输入方式，结合内部针对资源、实例、插件等多种功能封装，为用户带来一站式的使用体验。\n\n![1.png](https://official-media.sphere-ex.com/1_5bd2d7ea7f.png)\n\n### 产品优势\n\n**功能易用**\n\n提高了用户体验，避免了配置错误，用户不需要通过配置和命令操作 SphereEx 企业数据服务平台，易用性得到极大提升，实现了平台使用的“零”瓶颈。\n\n**功能全面**\n\n完善开源版本的管理能力，提供了对 SphereEx 企业数据服务平台从基础资源到插件能力的管控，形成了对周边插件实现统一管控的解决方案。\n\n**功能可视**\n\n原生提供可视化的监控数据面板，可以实时在线查看 SphereEx 企业数据服务平台的集群、实例、主机的监控数据。\n\n\n### 行业价值\n\n**本次 SphereEx-Console 的发布，是 SphereEx 在数据管理生态下一次关键尝试，弥补了 Apache ShardingSphere 过去在管理易用性、可视化界面等维度的不足。** SphereEx-Console 以可视的操作界面，极大降低了企业运维人员的学习成本；统一的数据管控平台，也极大提升了企业的运维管理效率。\n\n作为一家以开源项目 Apache ShardingSphere 为基本的商业化公司，SphereEx 将持续增强自身在数据治理、管控等方面的能力，配以周边完善的生态基础，为用户提供更加全面和完善的 Apache ShardingSphere 应用生态以及 SphereEx 产品矩阵能力。\n\n**[点击了解详情](https://sphere-ex.com/zh/products/console/)**","date":"2022-01-26","author":"SphereEx","excerpt":"提供企业级的平台管控能力","metaDescription":null,"createdAt":"2022-03-11T09:39:49.579Z","updatedAt":"2022-03-11T09:55:15.098Z","publishedAt":"2022-03-11T09:40:24.284Z","locale":"zh"}}]}}},{"id":22,"attributes":{"feature":false,"title":"F6 Automobile Technology’s Multimillion Rows of Data Sharding Strategy Based on  Apache ShardingSphere","content":"[F6 Automobile Technology](https://www.f6car.com/) is an Internet platform company focusing on the informatization of the automotive aftermarket. It helps automotive repair companies (clients) build their smart management systems to digitally transform the auto aftermarket.\n\nThe data of different auto repair companies will certainly be isolated from each other, so theoretically the data can be stored in different tables of different databases. However, fast-growing enterprises face increasing data volume challenges: sometimes total data volume in a single table may approach 10 million or even 100 million entries.\n\nThis issue definitely challenges business growth. Moreover, growing enterprises are now also planning to split their systems into many microservices based on domains or business types, and accordingly, different databases are vertically required for different business cases.\n\n## Why Did We Need Data Sharding?\n\nRelational databases are bottlenecks when it comes to storage capacity, connection count, and processing capabilities.\n\nFirst, we always prioritize database performance. When the data volume of a single table reaches tens of millions, and there are a relatively large number of query dimensions, system performance would still prove unsatisfactory even if we added more slave databases and optimize indexes. This meant it was time for us to consider data sharding.\n\nThe purpose of data sharding is to reduce database load stress and query time. Additionally, since a single database often has a limited number of connections, when Queries Per Second (QPS) indicator of the database is too high, database sharding is certainly needed to share connection stress.\n\nSecond, to ensure availability was another important reason. If unfortunately, an accident occurs in a single database, we’d likely lose all data and further affect all services. Database sharding can minimize risk and the negative impact on business services. Generally, when the data volume of a table is greater than 2GB or the number of data rows is greater than 10 million, not to mention that the data is also growing rapidly, we’d better use data sharding.\n\n## What’s Data Sharding?\n\nThere are four common types of data sharding in the industry:\n\nVertical table sharding: split big tables into small ones, field-based, which means less frequently used or relatively long fields are split into extended tables.\n\nVertical database sharding: business-based database sharding is used to solve performance bottlenecks of a single database.\n\nHorizontal table sharding: distribute tables’ data rows into different tables according to some rules in order to decrease the data volume of single tables and optimize query performance. In terms of the database layer, it still faces bottlenecks.\n\nHorizontal data sharding: based on horizontal table sharding, distribute data into different databases to effectively improve performance, lower stress of stand-alone machine and single databases, and break the shackles of I/O, connections, and hardware resources.\n\n## Excellent Data Sharding Solutions\n\n**1. [Apache ShardingSphere-JDBC](https://shardingsphere.apache.org/document/current/en/quick-start/shardingsphere-jdbc-quick-start/)**\n\n**Pros：**\n\n- Supported by an active open source community. Now, Apache ShardingSphere 5.0 version has been released and the development iteration speed is fast.\n- Proven efficacy by many successful enterprise application cases: big companies such as JD Technology and Dangdang.com have applied ShardingSphere.\n- Easy deployment: Sharding-JDBC can be quickly integrated into any project without extra services deployed.\n- Excellent compatibility: it can route to a single data node and perfectly support SQL.\n- Excellent performance and low loss: test results can be found on the Apache ShardingSphere website.\n\n**Cons:**\n\nPotential increase in operations and maintenance costs, and troublesome field changes and index creation after data sharding. To fix the issue, users need to deploy Sharding-Proxy that supports heterogeneous languages and is more friendly to DBAs.\n\nSo far, the project doesn’t support data shards dynamic migration yet. Therefore, feature implementation is required.\n\n**2. MyCat**\n\n**Pros:**\n\n- MyCat is a middleware placed between applications and databases to handle data processing and interactions. It cannot be perceived during development, and integrating MyCat does not cost much.\n- Use JDBC to connect databases such as Oracle, DB2, SQL Server, and MySQL.\n- Support multiple languages plus easy deployment and implementation across different platforms.\n- High availability and auto-switch triggered by a crash.\n\n**Cons:**\n\n- High operations and maintenance costs: to use MyCat, it’s required to configure a series of parameters plus HA load balancer.\n- Users have to independently deploy the service, which may increase system risks.\n\nOf course, there are similar solutions such as Cobar, Zebra, MTDDL, and TiDB but honestly, we didn’t spend much time researching other solutions, because we decided to use ShardingSphere as we felt it meets the company’s needs.\n\n## F6 Automobile Technology’s Overall Plan\n\nBased on our company’s business model, we chose Client ID as Sharding Key to ensure that work order data of one client is stored in the same single table of the same client-specific database. Therefore, performance loss caused by multi-table correlated queries is avoided; plus later, even if multi-databases sharding is required, cross-database transactions and cross-database JOIN can be avoided.\n\nAmong client ID databases, the type BIGINT(20) applies UID (Unique Identification Number, or we call it “gene” ) to ensure potential database scaling in the future; the last two digits of a client ID are its UID, so according to the double scaling rule, the maximum reaches 64 databases. The values of left bits can be used for table sharding, which can be split into 32 sharding tables.\n\nTake 10545055917999668983 as the client ID example and the rules are shown as follows:\n\n```\n105450559179996689 83\nTable sharding uid value % 32 database sharding uid value % 1\n```\nThe last two digits (i.e. 83) are used for database sharding, of which temporary data is only sharded into the library f6xxx, so the remainder is 0. Later, increasing data volume can be expanded to multiple libraries. The remaining value 105450559179996689 is used for table sharding. At first time, it is divided into 32 single tables so the modulo remainders correspond to the specific sharding table subscripts are 0~31.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/rdc5apgqko2381hhrvxw.png)\n \nGiven that the business system is growing and we adopt rapid iteration method to develop features step by step, we plan to shard tables first and then do database sharding.\n\nData sharding has a great impact on the system, so we need greyscale release— if unfortunately, an issue occurs, the system can quickly start roll-back, to ensure a functioning business system. The implementation details are given below:\n\n**Table Sharding**\n\n1. Switch from JDBC to Sharding-JDBC to connect data sources\n2. Decouple the write databases, and then migrate codes\n3. Synchronize historical data and incremental data\n4. Switch sharding tables\n\n**Database Sharding**\n\n1. Migrate the read-only databases\n2. Data migration\n3. Switch read-only databases\n4. Switch write-only databases\n\n## Table Sharding Details\n\n**_Number of Sharding Tables_**\n\nIn the industry, the data of a single table should usually be limited to 5 million rows, and the number of sharding tables should be a power of two to make them scalable. The exact number of sharding tables is calculated based on business development speed and future data increase as well as the future data archiving plan. After sharding table count and sharding algorithms are defined, it’s OK to assess the current data volume in each sharding table.\n\n**_Preparation_**\n\n**- Replace the database & auto table ID generator**\n\nAfter table sharding, we could no longer use the auto database ID generator anymore, so we had to find a feasible solution. We had two plans:\n\nPlan 1: Use other keys such as snowflake\n\nPlan 2: Implement an incremental component (database or Redis) all by ourselves\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/qc0qn1rn3d2261fn12ys.png)\n \nAfter comparing the two solutions and the business condition, we decided to choose Plan 2 and concurrently, provided a new comprehensive table-level ID generator solution.\n\n**- Check whether all requests are carried with shard keys**\n\nNow, the microservice traffic entrances include:\n\n- HTTP\n- Dubbo\n- XXLJOB scheduled job\n- Message Queue (MQ)\n\nAfter table sharding, to quickly locate data shards, all requests must carry their shard keys.\n\n**- Decoupling**\n\n1. Decouple business systems of each domain and use interfaces to interact with read and write data.\n\n2. Remove Direct Table JOIN and use interfaces instead.\n\nThe biggest problem brought by the decoupling is the distributed transaction problem: how to ensure data consistency. Usually, developers introduce distributed transaction components to ensure transaction consistency or they use compensation or other mechanisms to ensure final data consistency.\n\n_**Grayscale Release Plan**_\n\nIn order to ensure quick roll-back when problems caused by new feature releases occur, all online modifications are released step by step based on clients. Our grayscale release plan is shown as follows:\n\n**Plan 1:** Maintain two sets of Mapper interfaces: one uses Sharding-JDBC data sources to connect to databases while the other uses JDBC data sources to connect to databases. At the service layer, it’s necessary to select one of the two interfaces based on the decision workflow diagram below:\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/eivsoq9mms92ess7g0m1.png)\n \nHowever, the solution causes another problem: all codes visiting the Mapper layer have an `if else` branch, resulting in major business code changes, potential code intrusion, and harder code maintenance. Therefore, we found another solution and we call it Plan 2.\n\n**Plan 2 — Adaptive Mapper Selection Plan:** one set of Mapper interface is with two data sources and two sets of implementations. Based on the grayscale configuration, different client requests will go through different Mapper implementations, and one service corresponds to two data sources and two sets of transaction managers, and based on the grayscale configuration, different clients’ requests go to different transaction managers. Accordingly, we leverage multiple Mapper scanners of MyBatis to generate multiple mapperInterfaces, and concurrently generate a mapperInterface for wrapping. The wrapper class supports hintManagerto automatically select mappers; the transaction manager is similar to wrapper class generation. The wrapper class supports hintManager to automatically select various transaction managers to manage transactions. This solution actually avoids intrusion because for codes of the service layer, there is only one Mapper interface.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/obzvh9hgdwke71zbnpur.png)\n \n**_Data Source Connection Switch_**\n\nApache ShardingSphere already lists some grammars currently not supported by Sharding-JDBC on its website, but we still found the following problematic SQL statements that Sharding-JDBC parser cannot handle:\n\n- A subquery without a shard key.\n- Not support` Insert` statements whose values include cast ifnull now and other functions.\n- Not support` ON DUPLICATE KEY UPDATE`.\n- By default, select for update goes to the slave database (the issue has fixed since 4.0.0.RC3).\n- Sharding-JDBC does not support the statement ResultSet.first () of MySqlMapper with Optimistic Concurrency Control used to query vision.\n- No such statement for batch updates.\n- Even if `UNION ALL` does not support the grayscale release plan, we only need to copy a set of mapper.xml, and modify it based on the syntax of Sharding-JDBC before release.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/33j5pacig5eg1afdq4vs.png)\n\n**_Historical Data Sync_**\n\n[DataX](https://www.alibabacloud.com/help/en/doc-detail/126635.html) is Alibaba Group’s offline data synchronization tool that can effectively sync heterogeneous data sources such as MySQL, Oracle, SqlServer, Postgre SQL, HDFS, Hive, ADS, HBase, TableStore(OTS), MaxCompute(ODPS) and DRDS.\n\nThe data synchronization framework DataX can abstract the synchronization of different data sources as a Reader Plugin that reads data from the data source, and then as a Writer Plugin that writes data to the target. In theory, the DataX framework can support data synchronization of all data source types. Additionally, the DataX plugin ecosystem can allow every newly-added data source to immediately interact with the old data sources.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/fzcjfuckkoo0ho77w1w4.png)\n\n**_Verify Data Synchronization_**\n\n- Use timed tasks to compare the number of data of the original table and of the sharding table.\n- Use timed tasks to compare the values of key fields.\n\n**_Read/write Splitting and Table Sharding_**\n\nBefore read/write splitting, we needed to configure incremental data synchronization first.\n\n**-Incremental Data Synchronization**\n\nWe used another open source distributed database sync project named Otter to synchronize incremental data. Based on database incremental log parsing, otter can synchronize data of MySQL/Oracle databases of the local computer room or the remote computer room. To use Otter, we needed to pay extra attention to the following tips:\n\n- For MySQL databases, users must have binlog enabled, and set its mode as ROW.\n- The user must have the query permission of binlog so they need to apply for that as an otter user.\n- Now, the binlog of DMS database is stored for only 3 days. In Otter, users can define the starting position of binlog synchronization and the starting point of incremental synchronization by themselves: first, select slave-testDb on the SQL platform, and use the SQL statement “show master status” to query.\n\n> Note: the execution results of show master statusof master and slave may be different, so if you set it, you need to get the execution result of the master database. We think this function is really useful because when Otter’s data synchronization fails, we can reset points and synchronize again from the beginning.\n\n- When Otter is disabled, it will automatically record the last point of synchronization, and continue to synchronize data from this point next time.\n- Otter allows developers to define their own processing process. For example, we can configure data routing rules and control the direction of client data synchronization data from subtable to parent table or vice versa.\n- Disabling Otter will not invalidate the cache defined in the user-defined Otter processing process. To fix it, the solution is to modify the code comment and save it.\n\n- **Read/write Splitting Plan**\n\nOur greyscale switch plan is shown below:\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/r2ftt2x3pv1xwrbom1m6.png)\n\nWe chose the grayscale release solution, which means it was necessary to ensure real-time data updates in both subtables and parent tables. Therefore, all data was synchronized in two directions: for clients with grayscale release being on, reads and writes went to subtables and the data was synchronized to parent tables in real time via Otter, while for clients with grayscale release being off, reads and writes went to parent tables and the data is synchronized to subtables in real time via Otter.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/ic3nh61yipayjvsf259t.png)\n\n##  Database Sharding Details\n**_Preparations_**\n\n**- Primary Key**\nPrimary key should be auto increment type (sharding tables should be global auto increment) or access the only primary key numbering service (server_id independent of DB). Table auto increment primary key generation or uuid_short generated key require switching.\n\n**- Storage Procedure, Functions, Trigger and EVENT**\nTry to remove them first if present; if they cannot be removed, create them in advance in new databases.\n\n**- Data Synchronization**\nData synchronization uses DTS or sqldump (historical data) + otter (incremental data) for synchronization.\n\n**- Database Change Procedure**\nTo avoid potential performance and compatibility problems, database change plan must follow two criterion:\n\n- Greyscale switching: traffic is gradually switching to RDS (Alibaba Cloud Relational Database Service, a.k.a. RDS), allowing obervance of database performance at any time.\n- Quick Rollback: achieving quick reversion when problems occur with little impact on user experience.\n\n**Status Quo:** four application instances +one master db and two slave db\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/8l2szt3kc9ygpcu1o65k.png)\n\n**Step 1:** add a new application instance and switch it to RDS, write into or pass dms master database, and the data in dms master database will be synced to rds in real time\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/pv6r9qptfy7lsfpmsfzj.png)\n\n**Step 2:**add three more application instances, and cut 50% of the data to write into rds database \n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/cybdga96h0wnj5byix4r.png)\n \n**Step 3:** remove the four original instance traffic, and read them into rds instances while writing still goes into dms master database\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/e12d8o29ugwuz0l1xzcp.png)\n \n**Step 4:** switch the master database into rds, and rds data will be reversely synced to dms master database to make it easier for the quick rollback of the data\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/9c4v451ofi535xwx28nl.png)\n\n**Step 5:** Completion\n\n ![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/57jimac2lhn2gflcbdup.png)\n\nEach step mentioned above can be quickly rolled back through traffic switching so as to ensure the availability and stability of system.\n \n**Sharding & Scaling**\n\nWhen the performance of a single database reaches a plateau, we can scale out the database by modifying sharding database routing algorithms and migrating data.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/0rpthxwsficzwqfe2zn7.png)\n\nWhen the capacity of a single table reaches its maximum size, we can scale out the table by modifying sharding table routing algorithms and migrating data.\n\n ![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/86tuypkympqtcez27m4w.png)\n \n\n## FAQ\n**_Q: Sometimes, Otter receives binlog data but the data cannot be found in the database?_**\n\n**A:**To make our MySQL compatible with replicas of other non-transactional engines, we added binlog at the server layer. Binlog can record all engine modification operations, so it can support replication function for all engines. The problem is a potential inconsistency between redo log and binlog but MySQL uses its internal XA mechanism to fix the issue.\n\n**Step 1:**Not perform operations on InnoDB prepare, write/sync redo log and binlog.\n\n**Step 2:** First, write/sync Binlog and then InnoDB commit (commit in memory).\n\nOf course, group commit has been added since version 5.6. The development improves I/O performance to some degree, but it doesn’t change the execution order.\n\nAfter write/sync Binlog is done, the binlog has been written, so MySQL considers that the transaction has been committed and persisted (now, the binlog is ready to be sent to subscribers). Even if a database crashes, the transaction can still be recovered correctly after MySQL reboot. However, before this step, any operation failure may cause transaction rollback.\n\nInnoDB commit is centered on memory commit such as killing locks, read views related to multiversion concurrency control release. MySQL believes that no errors occur in this step — once an error really occurs, the database will crash — MySQL itself cannot handle the crash. This step does not have any logic that causes transaction rollback. In terms of program operations, only after this step is completed, the changes caused by the transaction can be shown through the API or queries at the client-side.\n\nThe reason why the problem may occur is that the binlog is sent first, and then db commit is done. We use query retries to fix this issue.\n\n**_Q： When it comes to multi-table queries, sometimes, why some tables cannot get data?_**\n\n**A:**The master/slave routing strategy of Sharding-JDBC is shown below:\n\nMaster databases are chosen in the following scenarios:\n\n- SQL statements that include lock such as select for update ( of Version 4.0.0.RC3);\n- Not SELECT statements;\n- Threads that have already gone to master databases;\n- Codes specify the requests for master databases.\n\nAlgorithms used to choose from multiple slave databases:\n\n- Polling Strategy\n- Load Balancer Strategy\n\nThe default is the polling strategy. However, one query may go to different slave databases, or it may go to the master library and slave databases, which occurs when there is time inconsistency between master-slave database latency or multi-slave latency.\n\n**_Q: How can we remove network traffic?_**\n**A:**\n\n- http: use nginx to remove upstream;\n- dubbo: leverage its qos module to execute offline/online command;\n- xxljob: manually enter execution IP of the executor to specify instances;\n- MQ: use the API provided by Alibaba Cloud to enable or disable consumer bean.\n\n## Apache ShardingSphere Open Source Project Links:\n\n[ShardingSphere Github](https://github.com/apache/shardingsphere)\n[ShardingSphere Twitter](https://twitter.com/ShardingSphere)\n[ShardingSphere Slack Channel](https://apacheshardingsphere.slack.com/ssb/redirect)\n","date":"2022-01-24","author":"SphereEx","excerpt":"F6 Automobile Technology is an Internet platform for the automotive aftermarket. It switched to sharding and chose ShardingSphere for its multimillion rows of data. ","metaDescription":"F6 Automobile Technology is an Internet platform company focusing on the informatization of the automotive aftermarket.","createdAt":"2022-09-26T07:41:17.269Z","updatedAt":"2022-09-27T07:00:17.977Z","publishedAt":"2022-09-27T07:00:17.973Z","locale":"en","caseStudiesType":{"data":{"id":9,"attributes":{"name":"Internet","createdAt":"2022-09-26T10:34:03.086Z","updatedAt":"2022-09-27T06:59:34.838Z","publishedAt":"2022-09-27T06:59:34.836Z","locale":"en"}}},"cover":{"data":{"id":342,"attributes":{"name":"F6 automobile.png","alternativeText":"F6 automobile.png","caption":"F6 automobile.png","width":720,"height":443,"formats":{"thumbnail":{"name":"thumbnail_F6 automobile.png","hash":"thumbnail_F6_automobile_c02eaa72b3","ext":".png","mime":"image/png","width":245,"height":151,"size":17.64,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_F6_automobile_c02eaa72b3.png"},"small":{"name":"small_F6 automobile.png","hash":"small_F6_automobile_c02eaa72b3","ext":".png","mime":"image/png","width":500,"height":308,"size":42.94,"path":null,"url":"https://official-media.sphere-ex.com/small_F6_automobile_c02eaa72b3.png"}},"hash":"F6_automobile_c02eaa72b3","ext":".png","mime":"image/png","size":67.32,"url":"https://official-media.sphere-ex.com/F6_automobile_c02eaa72b3.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T07:41:48.589Z","updatedAt":"2022-09-26T07:41:48.589Z"}}},"metaLogo":{"data":{"id":356,"attributes":{"name":"F6 logo.png","alternativeText":"F6 logo.png","caption":"F6 logo.png","width":1200,"height":600,"formats":{"thumbnail":{"name":"thumbnail_F6 logo.png","hash":"thumbnail_F6_logo_6330d93329","ext":".png","mime":"image/png","width":245,"height":123,"size":4.96,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_F6_logo_6330d93329.png"},"large":{"name":"large_F6 logo.png","hash":"large_F6_logo_6330d93329","ext":".png","mime":"image/png","width":1000,"height":500,"size":35.19,"path":null,"url":"https://official-media.sphere-ex.com/large_F6_logo_6330d93329.png"},"medium":{"name":"medium_F6 logo.png","hash":"medium_F6_logo_6330d93329","ext":".png","mime":"image/png","width":750,"height":375,"size":22.18,"path":null,"url":"https://official-media.sphere-ex.com/medium_F6_logo_6330d93329.png"},"small":{"name":"small_F6 logo.png","hash":"small_F6_logo_6330d93329","ext":".png","mime":"image/png","width":500,"height":250,"size":12.46,"path":null,"url":"https://official-media.sphere-ex.com/small_F6_logo_6330d93329.png"}},"hash":"F6_logo_6330d93329","ext":".png","mime":"image/png","size":32.65,"url":"https://official-media.sphere-ex.com/F6_logo_6330d93329.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:51:46.576Z","updatedAt":"2022-09-26T08:51:46.576Z"}}},"localizations":{"data":[]}}},{"id":23,"attributes":{"feature":false,"title":"Apache ShardingSphere & Keep: Asia’s Leading Workout & Fitness Trainer App Maker","content":"For our community’s first visit, we went to Asia’s leading workout & fitness trainer app maker “Keep” headquarters at Vanke Times Center and shared our technologies with developers at Keep Co.\n\nWay back in 2018, Keep had already deployed ShardingSphere capabilities such as sharding and read/write splitting in multiple application scenarios for its diversified lines of business.\n\nWith the release of Apache ShardingSphere 5.0, the concept of Database Plus and pluggable architecture has to some extent, reshaped the ShardingSphere ecosystem. During our visit, we conducted in-depth exchanges and discussions with our counterparts at Keep.\n\nKeep engineers expressed great interest in Apache ShardingSphere 5.0. At the event, Juan Pan, Apache ShardingSphere PMC and SphereEx CTO, presented a full overview of the initial ShardingSphere architecture, user-end access, community building and [the Database Plus concept.](https://www.infoq.com/articles/next-evolution-of-database-sharding-architecture/)\n\n## Database Plus: Freeing DBAs and Developers\n\nDatabase Plus is a design concept of the distributed database system. By building a standard layer and ecosystem for use and interaction above fragmented heterogeneous databases, and by multiplying and expanding computing capabilities (e.g. data sharding, data encryption and decryption), all the interaction between applications and databases is oriented to the standard layer built by Database Plus. This results in shielding the differentiated impact of database fragmentation on upper layer transactions.\n\nPan believes that the global database industry has boomed thanks in large to the following two reasons:\n\n**Business side demand:**\n\nIn terms of business, to ensure sustained visits and transactions growth, underlying databases must respond to requests as soon as possible. In addition, the breakup of microservices and subsequent modification of corresponding databases also generate demands from the business side.\n\n**Operations & maintenance side demand:**\n\nDBAs are responsible for running the whole business and data system, including data security, backup, distributed governance, and API smart monitoring of data clusters.\nIf the middle layer can meet transaction traffic and understand the request, DBAs can modify the request accordingly and then more operations can be performed. Therefore, we need to strike a balance between the two sides of demand to carry transaction traffic and support database capacity building, thus creating an efficient, collaborative ecosystem between the two sides.\n\nFollowing the version 5.0 release, the sharding function is no longer at the core of Apache ShardingSphere. Actually, sharding has been “downgraded” to a secondary function in the ShardingSphere ecosystem. Following the Database Plus concept, Apache ShardingSphere has built a pluggable architecture ecosystem, enabling the middle layer to achieve more value-added capabilities.\n\n![ShardingSphere’s Database Plus Architecture\n](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/c8kc9fdyp0n1kd3eo341.png)\n\nTaking data encryption and decryption as an example, we can see how they are generally done at the business layer because MySQL and other databases do not support encrypted algorithms themselves. This means encryption & decryption is only be achievable at application and business layers. However, this poses a problem if we consider complex online businesses that face considerable tasks when they upgrade their encryption algorithms.\n\nThe best solution to this problem is to encrypt and decrypt data at the middle layer. ShardingSphere-Proxy can be directly bound to databases, be placed at the middle layer between applications and underlying databases. ShardingSphere-proxy will feel “DB-like” by taking advantage of protocols that are compatible with different databases, thus deciding which node the SQL quest will fall on, letting upper layer applications be programmed for ShardingSphere, and shielding the impact of underlying databases. Data is in plaintext status when the transaction sends a request, but it will be encrypted after going through Proxy and decrypted when being retrieved.\n\n![ShardingSphere Encryption & Decryption Capability](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/hkesryh371m8tzlqct3c.png)\n\nAs such, the ShardingSphere encryption and decryption process can be separated from the existing system of applications and databases and can be linked with special encryption algorithms, especially in the cases where a cipher machine is involved. ShardingSphere capabilities such as encryption can free significant amounts of DBAs’ and developers’ time, allowing them to focus more on businesses.\n\n\n## Apache ShardingSphere Open Source Project Links:\n\n[ShardingSphere Github](https://github.com/apache/shardingsphere)\n[ShardingSphere Twitter](https://twitter.com/ShardingSphere)\n[ShardingSphere Slack Channel](https://apacheshardingsphere.slack.com/ssb/redirect)\n\n","date":"2022-01-10","author":"Yacine Si Tayeb","excerpt":null,"metaDescription":"Keep is a mobile fitness community that offers a variety of training programs for different groups of people.","createdAt":"2022-09-26T07:48:54.261Z","updatedAt":"2022-09-27T07:00:22.104Z","publishedAt":"2022-09-27T07:00:22.099Z","locale":"en","caseStudiesType":{"data":{"id":9,"attributes":{"name":"Internet","createdAt":"2022-09-26T10:34:03.086Z","updatedAt":"2022-09-27T06:59:34.838Z","publishedAt":"2022-09-27T06:59:34.836Z","locale":"en"}}},"cover":{"data":{"id":344,"attributes":{"name":"keep.png","alternativeText":"keep.png","caption":"keep.png","width":880,"height":347,"formats":{"thumbnail":{"name":"thumbnail_keep.png","hash":"thumbnail_keep_d033ce5311","ext":".png","mime":"image/png","width":245,"height":97,"size":18.27,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_keep_d033ce5311.png"},"medium":{"name":"medium_keep.png","hash":"medium_keep_d033ce5311","ext":".png","mime":"image/png","width":750,"height":296,"size":83.73,"path":null,"url":"https://official-media.sphere-ex.com/medium_keep_d033ce5311.png"},"small":{"name":"small_keep.png","hash":"small_keep_d033ce5311","ext":".png","mime":"image/png","width":500,"height":197,"size":47.39,"path":null,"url":"https://official-media.sphere-ex.com/small_keep_d033ce5311.png"}},"hash":"keep_d033ce5311","ext":".png","mime":"image/png","size":24.11,"url":"https://official-media.sphere-ex.com/keep_d033ce5311.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T07:53:26.070Z","updatedAt":"2022-09-26T07:53:26.070Z"}}},"metaLogo":{"data":{"id":357,"attributes":{"name":"keep logo.png","alternativeText":"keep logo.png","caption":"keep logo.png","width":205,"height":204,"formats":{"thumbnail":{"name":"thumbnail_keep logo.png","hash":"thumbnail_keep_logo_ecc24449b4","ext":".png","mime":"image/png","width":157,"height":156,"size":13.7,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_keep_logo_ecc24449b4.png"}},"hash":"keep_logo_ecc24449b4","ext":".png","mime":"image/png","size":18,"url":"https://official-media.sphere-ex.com/keep_logo_ecc24449b4.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T09:00:55.470Z","updatedAt":"2022-09-26T09:00:55.470Z"}}},"localizations":{"data":[]}}},{"id":24,"attributes":{"feature":false,"title":"Apache ShardingSphere & Sohu","content":"The Apache ShardingSphere community team visited Sohu.com headquarters for in-depth discussions with their counterparts at the internet giant. At the meet-up, Juan Pan, Apache ShardingSphere PMC and SphereEx CTO, elaborated on Apache ShardingSphere’s capabilities.\n\nSohu, Inc., an integrated Internet company with a rich history and comprehensive lines of business, uses ShardingSphere products in their multiple lines of business, such as social media and video services.\nDriven by ShardingSphere-JDBC and with ShardingSphere-Proxy as the container management platform, ShardingSphere provides support for Sohu in all aspects, helping Sohu relieve database traffic stress.\n\nAt the event, engineers from Sohu.com expressed great interest in Apache ShardingSphere 5.0. Juan Pan presented a full overview of ShardingSphere’s architecture, client end access, community building and the [Database Plus](https://dzone.com/articles/apache-shardingsphere) concept.\n\n## Navigate & Solve Industry Challenges\n\nDue to the complexity of enterprise database systems, and the heavy cost of databases, service providers are supposed to provide enterprises with mainstream database products as well as products and services that can meet multiple needs.\n\nFacing diverse users' needs and increasingly diversified products in the database market, one viable solution could be database deployment and functions with higher flexibility and scalability.\n\nBy creating upper-level standards and ecosystems for heterogeneous databases, Apache ShardingSphere provides diversified functions that can precisely meet enterprise needs. As an ecosystem consisting of multiple adapters, by using a hybrid deployment model of ShardingSphere-JDBC and ShardingSphere-Proxy, Apache ShardingSphere enables users to configure sharding strategies through one console and to flexibly create application systems that suit the needs in different scenarios. This allows engineers more freedom to build system architectures that could best suit the needs of their ongoing projects.\n\n![ShardingSphere’s Deployment Architecture](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/qmtr376g8hbr9ssz37z0.png)\n\nCentering around connections, incremental and pluggable features, and based on a pluggable architecture, Apache ShardingSphere creates a powerful kernel architecture in the form of a micro-kernel.\n\nBased on its powerful kernel capabilities, Apache ShardingSphere products can provide users with ideal solutions for distributed database, data security, database gateway, and [full-link stress testing](https://shardingsphere.medium.com/full-link-online-stress-testing-for-production-database-apache-shardingsphere-shadow-database-84f7cba56f99), helping increase the efficiency of enterprises and users.\n\n![Overview of Apache ShardingSphere’s Functions](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/3fsgj8f7tf6b1rcq0ufl.png)\n\nWhen it comes to future optimization, Pan believes that some deficiencies remain in Apache ShardingSphere’s performance and this is particularly true for ShardingSphere-Proxy’s adaptor. \n\nIn the future, the community will double down on improving its kernel, minimizing business and data performance loss of the Proxy, improving routing and rewriting logic, reducing creating database objects, and avoiding excess young GC.\n\n**Apache ShardingSphere Open Source Project Links:**\n\n[ShardingSphere Github]\n(https://github.com/apache/shardingsphere/issues?page=1&q=is%3Aopen+is%3Aissue+label%3A%22project%3A+OpenForce+2022%22)\n\n[ShardingSphere Twitter](https://twitter.com/ShardingSphere)\n\n[Contributor Guideline](https://shardingsphere.apache.org/community/cn/contribute/)\n\n","date":"2022-01-10","author":"Yacine Si Tayeb","excerpt":"Sohu offers a network of web properties and community-based products and services for Chinese users.","metaDescription":null,"createdAt":"2022-09-26T07:50:57.763Z","updatedAt":"2022-09-27T07:00:26.583Z","publishedAt":"2022-09-27T07:00:26.580Z","locale":"en","caseStudiesType":{"data":{"id":6,"attributes":{"name":"Media","createdAt":"2022-09-26T10:29:40.567Z","updatedAt":"2022-09-27T06:59:40.008Z","publishedAt":"2022-09-27T06:59:40.006Z","locale":"en"}}},"cover":{"data":{"id":343,"attributes":{"name":"Sohu.jpg","alternativeText":"Sohu.jpg","caption":"Sohu.jpg","width":880,"height":460,"formats":{"thumbnail":{"name":"thumbnail_Sohu.jpg","hash":"thumbnail_Sohu_ba5ad984c2","ext":".jpg","mime":"image/jpeg","width":245,"height":128,"size":3.23,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_Sohu_ba5ad984c2.jpg"},"medium":{"name":"medium_Sohu.jpg","hash":"medium_Sohu_ba5ad984c2","ext":".jpg","mime":"image/jpeg","width":750,"height":392,"size":15.57,"path":null,"url":"https://official-media.sphere-ex.com/medium_Sohu_ba5ad984c2.jpg"},"small":{"name":"small_Sohu.jpg","hash":"small_Sohu_ba5ad984c2","ext":".jpg","mime":"image/jpeg","width":500,"height":261,"size":9.06,"path":null,"url":"https://official-media.sphere-ex.com/small_Sohu_ba5ad984c2.jpg"}},"hash":"Sohu_ba5ad984c2","ext":".jpg","mime":"image/jpeg","size":18.61,"url":"https://official-media.sphere-ex.com/Sohu_ba5ad984c2.jpg","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T07:53:26.009Z","updatedAt":"2022-09-26T07:53:26.009Z"}}},"metaLogo":{"data":{"id":358,"attributes":{"name":"sohu logo.png","alternativeText":"sohu logo.png","caption":"sohu logo.png","width":317,"height":159,"formats":{"thumbnail":{"name":"thumbnail_sohu logo.png","hash":"thumbnail_sohu_logo_46a734f7df","ext":".png","mime":"image/png","width":245,"height":123,"size":25.89,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_sohu_logo_46a734f7df.png"}},"hash":"sohu_logo_46a734f7df","ext":".png","mime":"image/png","size":10.38,"url":"https://official-media.sphere-ex.com/sohu_logo_46a734f7df.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T09:04:52.675Z","updatedAt":"2022-09-26T09:04:52.675Z"}}},"localizations":{"data":[]}}},{"id":25,"attributes":{"feature":true,"title":"Apache ShardingSphere & iQiyi: What's Database Mesh?","content":"> _To further understand application scenarios, enterprises’ needs, and improve dev teams’ understanding of Apache ShardingSphere, SphereEx launched the “Enterprise Visits” series. The second stop took us to the leading online streaming platform in Asia, iQiyi._\n\nIn November 2021, a team from the ShardingSphere community visited the innovation center of iQiyi for in-depth interactions and discussions with their counterparts from Beijing and Shanghai. During the meetup, Zhang Liang, Apache ShardingSphere PMC Chair and SphereEx Founder provided details on the latest Apache ShardingSphere community initiatives, its future development, and Database Mesh.\n\nDuring the meetup, iQiyi was especially interested in the capabilities and future plans of ShardingSphere and Database Mesh. here are some key takeaway questions from the visit:\n\n**Q: How does ShardingSphere manage its encryption keys?**\n\n[Encryption](https://shardingsphere.apache.org/document/current/en/features/encrypt/) key management is a capability subordinate to the encryption algorithm. When ShardingSphere needs an encryption key in its decryption, the key could be passed through properties configured by the encryption algorithm. In principle, encryption key management is not a capability of ShardingSphere but the job of the encryption algorithm.\n\n**Q:Is there a built-in cryptographic algorithm in ShardingSphere? Or is it done through outside cooperation?**\n\nStrictly speaking, ShardingSphere is not equipped with built-in capability. That’s because the pluggable architecture of ShardingSphere is divided into multiple layers, and the encryption module itself is only a capability of L2 of the pluggable architecture.\n\nThe encryption module only defines the top-layer encryption connector and the detailed encryption algorithm achieves the pluggable part of the connector. Currently, ShardingSphere supports several of the most commonly used open source algorithms, but it does not support any national cryptographic algorithm in particular.\n\nIntegrating an encryption algorithm is easy for developers: instead of changing source codes, all they need is a ShardingSphere encryption algorithm connector. ShardingSphere’s data encryption process is completely transparent to users and very easy to use.\n\n**What’s the future plan for Database Mesh?**\n\nDatabase Mesh is another ShardingSphere initiative. We are now planning to integrate Database Mesh through Sidecar. Sidecar will be used to manage traffic, while ShardingSphere will be used to manage to compute, and the underlying databases’ nodes to manage storage.\n\nSidecar’s and ShardingSphere functions overlap in part, but not completely. ShardingSphere has more than 190 modules, therefore it’s not possible to completely replace ShardingSphere with Sidecar. That’s why ShardingSphere empowers some lightweight capabilities to Sidecar, and Sidecar is equipped with the capabilities to buffer some traffic and requests.\n\nThe database computing capability is still processed by ShardingSphere, as it would consume too many application resources to run heavyweight computation via Sidecar for the distributed transaction, query and optimization of heterogeneous databases.\n\nIn the future, other Sidecar capabilities, such as management, SQL auditing and authorities, will be developed. In this way, Sidecar and ShardingSphere will have their own focuses, providing a better solution. \n\n**Sidecar is equivalent to a data plane. Will you combine the control plane with [Istio](https://istio.io/) or will you build the control plane separately?**\n\nShardingSphere uses DistSQL (Distributed SQL) to create rules for sharding, encryption and so on. ShardingSphere is already equipped with the control plane capability of Sidecar. Any capability achieved on Sidecar can be controlled by ShardingSphere’s [DistSQL](https://opensource.com/article/21/9/distsql). In the future, we will consider integrating Sidecar into the Istio ecosystem.\n\nSo far, Apache ShardingSphere with the release of a version of 5.0 has officially repositioned itself in a new field with the Database Plus concept at its core. In addition to the powerful incremental capability at the upper layer of databases, the Database Plus architecture also opens a highly extensible database ecosystem to developers and users, charting a new future development course for Apache ShardingSphere.\n\n**Apache ShardingSphere Open Source Project Links:**\n\n[ShardingSphere Github]\n(https://github.com/apache/shardingsphere/issues?page=1&q=is%3Aopen+is%3Aissue+label%3A%22project%3A+OpenForce+2022%22)\n\n[ShardingSphere Twitter](https://twitter.com/ShardingSphere)\n\n[Contributor Guideline](https://shardingsphere.apache.org/community/cn/contribute/)","date":"2022-01-10","author":"Yacine Si Tayeb","excerpt":"Also known as the Netflix of Asia, iQiyi is an ad-supported television and movie portal providing fully-licensed, high-definition media content.","metaDescription":null,"createdAt":"2022-09-26T07:52:25.745Z","updatedAt":"2022-09-28T02:38:53.315Z","publishedAt":"2022-09-27T07:00:31.240Z","locale":"en","caseStudiesType":{"data":{"id":9,"attributes":{"name":"Internet","createdAt":"2022-09-26T10:34:03.086Z","updatedAt":"2022-09-27T06:59:34.838Z","publishedAt":"2022-09-27T06:59:34.836Z","locale":"en"}}},"cover":{"data":{"id":389,"attributes":{"name":"iqiyi2.jpg","alternativeText":"iqiyi2.jpg","caption":"iqiyi2.jpg","width":500,"height":360,"formats":{"thumbnail":{"name":"thumbnail_iqiyi2.jpg","hash":"thumbnail_iqiyi2_3b19419f07","ext":".jpg","mime":"image/jpeg","width":217,"height":156,"size":5.22,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_iqiyi2_3b19419f07.jpg"}},"hash":"iqiyi2_3b19419f07","ext":".jpg","mime":"image/jpeg","size":12.26,"url":"https://official-media.sphere-ex.com/iqiyi2_3b19419f07.jpg","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-28T02:38:45.900Z","updatedAt":"2022-09-28T02:38:45.900Z"}}},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":6,"attributes":{"feature":true,"title":"Apache ShardingSphere & JD Baitiao: Story of an Implementation Journey","content":">Apache ShardingSphere has become the best solution for JD Baitiao’s very large storage scenarios and scale-up. JD’s shopping festival weighed very much on our big data storage. I mean, like hundreds of millions of data sets.\n>\n>JD Baitiao, an internet credit products provider launched in 2014, has always been challenged by data explosion. Every shopping festival is like a technical test for us, but every time the technical staff professionals take proactive strategic initiatives in data architecture development to ensure everything goes smoothly.\n>\n>— Dongfang Zhang，JD Batiao R&D Director\n\n## The Baitiao Data Architecture Evolution\n\nLaunched in Feb. 2014, JD Baitiao’s data architecture has already evolved for several times because of rapid increase in data volume and rapid business growth.\n\n### From 2014 to 2015\n\nBaitiao first adopted the Solr + HBase solution to connect core business systems and other systems with key databases. Solr indexing is able to achieve fast search responses and HBase is used to store a massive amount of data.\n\n- A Solr cluster can help the core database reduce reads and writes stress.\n- However, Solr is not perfect: it has unsatisfying extensibility and intrusions.\n\n### From 2015 to 2016\n\nThe company chose NoSQL to store monthly data tables in Mongo DB clusters, temporarily meeting the demand for importing/exporting massive data in payment settlement scenarios.\n\n- The change made it quite efficient to query hot data. In addition, thanks to such unstructured data storage, staff can easily modify table structures.\n- Nevertheless, the plan was still problematic: bad extensibility, intrusions, and memory hog.\n\n### From 2016 to 2017\n\nGrowing business and growing data. Tens of billions of data volume imposed more performance and volume pressure on MongoDB. JD Baitiao’s Big Data Platform used DBRep and MySQL Slave to capture data changes and store the information data in its message center, to later be written to ES and HBase.\n\n- This plan was actually much better. It focused on real-time data and improved extensibility.\n- However, Baitiao faced high costs to maintain its code because of the data sharding problem in its architecture.\n\nThe architecture evolution of JD Baitiao is just an illustration of rapidly growing Internet consumer finance. All the methods Baitiao adopted in the past were not comprehensive, indicating that every solution would be outdated soon.\n\n## Time to Adopt a Decoupled Architecture\n\nTo ensure good system performance even with rapidly increasing data volumes, the technical team first chose the sharding architecture pattern that not only guaranteed excellent performance, but also kept code under control.\n\n### The data striping plan was actually based on application architecture:\n\nThe product continuously developed but the early solution had now become one of the biggest problems. The old data sharding plan makes code even more complex and increases maintenance expenditure. Developers have to spend much time adjusting sharding every time when the application upgrades, so they cannot concentrate on their own developments. In fact, tight coupling is to blame for the problem.\n\nTherefore, the team decided to use a mature sharding component that simplifies system upgrades and architecture changes. To compare Baitiao’s shard and ShardingSphere shard, their differences are shown in the following table.\n\n![jd_en_1.png](https://official-media.sphere-ex.com/jd_en_1_eb365b4a93.png)\n\nFor Baitiao, decoupling was the next task.\n\nApparently, JD Baitiao‘s data architecture will experience a brand-new journey to decoupling. The following three directions definitely fueled its transformation:\n\n**Concentration:** instead of database sharding built in its architecture, it applied a sharding component to place more energy on it own product development.\n\n**Easier Upgrade:** it used a decoupled architecture to simplify the R&D process of system upgrades.\n\n**Future-Oriented Plan:** it aims to improve system extensibility so that Baitiao will be more capable of confidently holding large online shopping festivals like the 618 Shopping Festival and Double 11 Shopping Festival (also known as Singles Day on November 11th).\n\nJD Baitiao is a huge business, so its business scenario is truly related to finance, high concurrency, and massive data volumes. Baitiao’s sharding component must have the following features:\n\n1. Be a mature product\n2. Have excellent performance\n3. Be able to handle big data\n4. Have an extensible architecture\n\n## The Apache ShardingSphere Solution\n\nThe lightweight Java framework ShardingSphere-JDBC is Apache ShardingSpheres’s first product providing services such as a Java Database Connectivity (JDBC) API. ShardingSphere JDBC uses jar archive file installation package and allows the client-side to directly connect to databases. So, it requires no extra deployment dependencies. It is like an enhanced JDBC driver fully compatible with JDBC and ORM frameworks.\n\nShardingSphere — JDBC provides the following features, making it the best solution in the Baitiao scenario.\n\nMature Product: ShardingSphere — JDBC is a mature product developed for years, and its open source community is very active.\n\nExcellent Performance: its micro-kernel and lightweight design hardly hinder performance.\n\nMinimum Changes: it supports native MySQL protocol and minimizes R&D workload.\n\nExtensibility: users can combine with the migration sync component to easily extend data.\n\n![jd_en_2.png](https://official-media.sphere-ex.com/jd_en_2_c92fb9e5ef.png)\n\nAfter being systematically tested for several times, Apache ShardingSphere became JD Baitiao’s first choice of data sharding middleware. The cooperation started at the end of 2018.\n\n## Product Adaptation\n\nTo provide Baitiao with better services and support the business, Apache ShardingSphere has made many improvements to its product features and performance during the implementation process. At the same time, the user case helps optimize the product in return.\n\n### Upgrade the SQL Engine\n\nBaitiao’s business logic is extremely complicated; its diversified scenario demands require the best SQL compatibility levels. Thus, Apache ShardingSphere restructures the SQL parser module to support more SQLs.\n\n- Single data node routing: 100% SQL compatibility\n- Multi data node routing：comprehensively support DML, DDL, DCL, TCL and partly DAL. Support features such as pagination, deduplication, ranking, grouping, aggregation and correlated query.\n\n### Distributed Key\n\nApache ShardingSphere provides built-in distributed key generators, such as UUID, and SNOWFLAKE. It also provides its distributed key generation API so that users can develop custom key generation algorithms to meet their special needs.\n\n### Shard Key Value Injection\n\nIf a SQL has no shard condition, Apache ShardingSphere uses ThreadLoad to manage shard key value, and users can program and add a shard condition to HintManager, making the condition only effective in the current thread. That’s so-called SQL zero intrusion.\n\nAdditionally, Apache ShardingSphere continues to optimize its other features in order to meet Baitiao’s need for high performance such as：\n\n- SQL parsing result cache\n- JDBC metadata cache\n- Bind table & Broadcast table\n- Automated execution engine & Stream merge\n\nJD Baitiao & ShardingSphere teams joint efforts amde all indicators of the product live up to their expectation. The resulting final performance is almost the same as that of a native JDBC.\n\n![jd_en_3.png](https://official-media.sphere-ex.com/jd_en_3_582e2afe89.png)\n\n## Cutover\n\nApache ShardingSphere uses its custom HASH strategy to shard data, effectively avoiding the hot data problem. The total of data nodes almost reaches ten thousand. The cutover process lasts for about 4 weeks.\n\n1. After DBRep reads data, Apache ShardingSphere synchronizes the data to the target database cluster.\n2. Two clusters run together. After a data migration, Baitiao uses its own tool to verify business and data.\n\nDBRep is fundamental to the product design of ShardingSphere-Scaling. The automation feature of Scaling definitely facilitates migration and scale-up.\n\n## Apache ShardingSphere Brings Many Benefits\n\n### Simplified Upgrade Path\n\nThe decoupled architecture effectively simplifies a technology stack required in a system upgrade, so the developer team no longer need to worry about table sharding. Instead, they can focus on the business itself. In brief, ShardingSphere greatly helps Baitiao optimize the upgrade path.\n\n### Save R&D Cost\n\nBaitiao directly uses the mature product Apache ShardingSphere, so they do not need to reinvent the wheel and save a lot of time and energy.\n\n### Improve Architecture Extensibility\n\nBaitiao also uses the sync migration component Scaling that makes system scale-up even more flexible. It’s the company’s secret for online shopping festival successes.\n\n## Summary\n\nThe growth of JD Baitiao’s business greatly stimulates its data architecture upgrades. This time, it chose Apache ShardingSphere to decouple its architecture and make future upgrades less complicated. Thank to this from now on, developers only have to concentrate on their own product optimization because its data architecture is extensible enough. The case is really a good example of ShardingSphere application in the consumer finance scenario.\n\nNow, there are more and more internet credit payment models. In the future, Apache ShardingSphere in collaboration with JD Technology will explore more business scenarios, and contribute finance-related technological innovations, further improving Internet finance.\n\n### About Apache ShardingSphere\n\nShardingSphere is one of the Apache Software Foundation Top-Level Open Source projects, used by over 170 enterprises worldwide, across various verticals such as finance, e-commerce, cloud services, tourism, logistics, education, and entertainment. Its GitHub community has so far cumulated over 14,000 stars.\n\nWe welcome more technical professionals to write articles and share their experiences and thoughts. If you are interested, feel free to contact us:\n\nShardingSphere Github: https://github.com/apache/shardingsphere\n\nShardingSphere Twitter: https://twitter.com/ShardingSphere\n\nShardingSphere Slack Channel: apacheshardingsphere.slack.com","date":"2021-12-01","author":"SphereEx","excerpt":"Apache ShardingSphere has become the best solution for JD Baitiao’s very large storage scenarios and scale-up. JD’s shopping festival weighed very much on our big data storage. I mean, like hundreds of millions of data sets.","metaDescription":"JD Technology Group is a business subgroup of JD Group that focuses on technology as an industry service, and is committed to providing technical products and solutions for enterprises, financial institutions, governments and other types of customers across the value chain.","createdAt":"2021-12-12T12:30:44.957Z","updatedAt":"2022-09-26T09:16:15.245Z","publishedAt":"2021-12-12T12:35:08.683Z","locale":"en","caseStudiesType":{"data":{"id":2,"attributes":{"name":"E-commerce","createdAt":"2021-12-11T23:02:08.395Z","updatedAt":"2021-12-11T23:02:11.020Z","publishedAt":"2021-12-11T23:02:10.947Z","locale":"en"}}},"cover":{"data":{"id":345,"attributes":{"name":"JDT.png","alternativeText":"JDT.png","caption":"JDT.png","width":720,"height":540,"formats":{"thumbnail":{"name":"thumbnail_JDT.png","hash":"thumbnail_JDT_2704eac855","ext":".png","mime":"image/png","width":208,"height":156,"size":8.85,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_JDT_2704eac855.png"},"small":{"name":"small_JDT.png","hash":"small_JDT_2704eac855","ext":".png","mime":"image/png","width":500,"height":375,"size":28.29,"path":null,"url":"https://official-media.sphere-ex.com/small_JDT_2704eac855.png"}},"hash":"JDT_2704eac855","ext":".png","mime":"image/png","size":14.08,"url":"https://official-media.sphere-ex.com/JDT_2704eac855.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:02:50.084Z","updatedAt":"2022-09-26T08:02:50.084Z"}}},"metaLogo":{"data":{"id":10,"attributes":{"name":"jd_logo.png","alternativeText":"jd_logo.png","caption":"jd_logo.png","width":300,"height":44,"formats":{"thumbnail":{"name":"thumbnail_jd_logo.png","hash":"thumbnail_jd_logo_591b2e5e14","ext":".png","mime":"image/png","width":245,"height":36,"size":6.43,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_jd_logo_591b2e5e14.png"}},"hash":"jd_logo_591b2e5e14","ext":".png","mime":"image/png","size":2.36,"url":"https://official-media.sphere-ex.com/jd_logo_591b2e5e14.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2021-12-12T12:04:06.233Z","updatedAt":"2021-12-14T07:19:09.532Z"}}},"localizations":{"data":[{"id":1,"attributes":{"feature":true,"title":"Apache ShardingSphere 在京东白条场景的落地之旅","content":"\n>京东白条使用 Apache ShardingSphere 解决了千亿数据存储和扩容的问题，为大促活动奠定了基础。\n>\n>2014 年初，“京东白条”作为业内互联网信用支付产品，数据量爆发式的增长，每一次大促备战都是对技术人员的考验，每一次的战略转型驱动着数据架构的成长。\n>\n>— 张栋芳，京东白条研发负责人\n\n### 项目背景\n\n自 2014 年 2 月京东白条业务上线起，为满足快速发展的业务和激增的海量数据，白条的数据架构经历了数次演进。\n\n白条数据架构的演进间接地反应了互联网消费金融的飞速发展，也说明了每一种解决方案在不同背景下都有不同的保质期。\n\n### 业务挑战\n\n随着产品升级迭代，早期的解决方案演变成为了眼前的问题，通过业务框架实现的数据分片方案导致业务代码复杂度增加、维护成本不断攀升，紧耦合的弊端原形毕露，应用每次升级都需要投入较多的精力对分片做相应调整，研发人员难以专注于业务本身。\n\n技术团队经权衡后开始考虑使用成熟的分库分表组件来承担这部分工作，让业务系统升级和架构调整不再复杂。\n\n### 解决方案\n\nShardingSphere-JDBC 是 Apache ShardingSphere 的第一款产品，它定位为轻量级 Java 框架，在 Java 的 JDBC 层提供的额外服务。它使用客户端直连数据库，以 jar 包形式提供服务，无需额外部署和依赖，可理解为增强版的 JDBC 驱动，完全兼容 JDBC 和各种 ORM 框架。\n\nShardingSphere-JDBC 的以下特点能够很好地满足白条业务场景：\n\n- 产品成熟：经数年打磨产品成熟度高，且社区活跃；\n- 性能良好：微内核、轻量化的设计，性能损耗极小；\n- 改造量小：支持原生的 MySQL 协议，研发工作量小；\n- 扩展灵活：搭配使用迁移同步组件轻松实现数据扩展。\n\n经内部大量系统性验证之后，Apache ShardingSphere 成为了京东白条数据分片中间件的首选方案，2018 年底正式开始对接。\n\n![jd_zh_1.png](https://official-media.sphere-ex.com/jd_zh_1_fac28cd6ec.png)\n\n### 用户收益\n\n- 简化升级路径\n\n通过架构解耦，业务系统升级所涉及技术栈得到有效缩短，研发团队不再需要关注分表设计，精力全部聚焦于业务本身，升级路径得到大幅度优化；\n\n- 节省研发力量\n\n引入成熟的 Apache ShardingSphere 无需重新开发分表组件，在简化业务升级路径的基础上节省了大量研发力量；\n\n- 架构灵活扩展\n\n搭配使用 Scaling 同步迁移组件从容面对“618”和“11.11”等大型活动，系统灵活扩容。\n\n**SphereEx，让数据与应用连接更简单。**\n\n","date":"2021-12-01","author":"SphereEx","excerpt":"京东白条使用 Apache ShardingSphere 解决了千亿数据存储和扩容的问题，为大促活动奠定了基础。","metaDescription":"京东科技集团是京东集团旗下专注于以技术为产业服务的业务子集团，致力于为企业、金融机构、政府等各类客户提供全价值链的技术性产品与解决方案。","createdAt":"2021-12-11T23:10:03.172Z","updatedAt":"2021-12-14T07:22:15.304Z","publishedAt":"2021-12-11T23:10:07.307Z","locale":"zh"}}]}}},{"id":7,"attributes":{"feature":false,"title":"eHualu Story’s Data Processing Shortcut","content":"> ShardingSphere has greatly simplified sharding development and maintenance, which has played a very important role in our rapid product release. We estimate that ShardingSphere has saved us at least 4 months of R&D costs.\n> \n> –– Shi Moxuan, Technical Director，E-Hualu\n\nAt the beginning of this year, E-hualu officially launched its cloud service product “Hulu App” that targets non-business users. Since then, its architecture’s data stress has increased on a daily basis.\n\nThe R&D team of Hulu App chose to use ShardingSphere’s sharding features and scale their data out. The flexible and agile features of ShardingSphere met the requirements of Hulu App. It requires data extensibility, and needs to prevent the team from “reinventing the wheel”.\n\nShardingSphere can simplify sharding development and maintenance to the greatest extent. As the service is growing, the work will be increasingly complicated.\n\n### Hulu App’s growth pressure: feature extension and new services\n\nThe high requirement for data storage and computing is in the genes of the Hulu App technical team.\n\nBecause Hulu App is rapidly growing, its service and feature changes are relatively frequent. Therefore its technical team needs to adapt quickly when a front-end service changes. As its users and service data volume are growing rapidly, its databases are faced with greater pressure.\n\nThe official Hulu App was released on May 17, 2020, and then its user data and service volume also showed a rapid growth trend. Therefore it was inevitable to scale out its databases for many times. At the same time, with business requirements changing very quickly, new challenges have emerged:\n\n- How to ensure feature extensibility\n\nBecause of increasing users and developing product versions, its user data has exploded. The storage and computing capability of its old architecture has been severely challenged. Therefore, Hulu App requires that its back-end data processing platform shall have both extensible features, and flexibility.\n\n- How to improve efficiency\n\nFaced with its changeable services, the R&D team of Hulu App needs to make rapid adjustments when necessary, so as to make its architecture adapt to its services. A highly-flexible, extensible data architecture can greatly improve development efficiency of the team. On the other hand, because of such large amounts of data, database retrieval efficiency will inevitably have problems such as delays, slow read and write. These problems will ultimately affect user experience.\n\n- How to facilitate feature release\n\nAll new products will face problems like frequent new feature release, bringing the scheduled release date forward, when they are just released. These problems pose a great challenge to a development team. At first, Hulu’s development team planned to base on their service demands and create their own sharding plan all by themselves. However, they don’t have enough time to do that, so it’s urgent for them to choose a new sharding plan.\n\n- How to guarantee system stability\n\nA newly-added technology will greatly challenge system stability. It’s especially true for bottom-level technologies, most of which can influence platform services and probably challenge system stability. So what Hulu App needs is a data application product that does not greatly stress databases, but quickly adapts to them and that is also highly stable.\n\n### Leverage ShardingSphere’s features to Create a Flexible, Highly-Available Data Architecture Solution\n\nHulu App’s team studied their needs and spent two weeks evaluating ShardingSphere and other similar solutions. They had evaluative indicators like product feature, maturity, stability and performance. At last, they chose ShardingSphere because of its powerful features, its support and its high maturity can fully meet their needs.\n\nThe Hulu team deployed ShardingSphere above Alibaba Cloud’s RDS. Instead of adjusting ontology databases, the Hulu team prefers to conduct data governance above its databases. ShardingSphere can effectively improve Hulu’s data architecture, because it can achieve three things: high scalability brought by its extensible architecture, closely adaptive to Hulu’s service. ShardingSphere brings Hulu with obvious improvement:\n\n#### “High scalability” of the pluggable architecture\n\nDue to its service characteristics, Hulu App’s limited storage space is consumed very quickly, and the problem gradually begins to affect response efficiency at the front end. Therefore, the team can quickly develop enhanced features by using ShardingSphere, which can provide optimized solutions for its subsequent architectural adjustments. The example strengthens the ShardingSphere’s advantage in sharding.\n\n#### Closely adaptive to service\n\nThe smaller the architecture change is, the more controllable it is for developers. ShardingSphere, an ecosystem located above databases, is closer to service and it deployment is even more lightweight. So it is undoubtedly the best solution to solve the contradiction between the front-end service changes and back-end architecture adjustment of Hulu App. In addition, in terms of flexibility, ShardingSphere supports instant achievement of relevant configurations, which greatly helps the team save a lot of sharding-related work. So it plays a very important role in Hulu App’s quick releases.\n\n#### “Zero attack” to the service architecture\n\nHulu App chose ShardingSphere’s Proxy deployment mode. It can manage the real database cluster through Proxy without. replacing underlying databases. It basically completed the separation of service and data at the architectural level without any changes. So it can avoid the risks caused by database changes, such as unavailable service and long stable cycle. In addition, the non-status mode of ShardingSphere hardly has any perceptible impact on users at the front end and the service layer does not need to pay attention to the data storage method .\n\nSo, Hulu App is a product with tight go-live schedule and fast feature iteration. ShardingSphere-Proxy can reuse original databases and help the Hulu development team to develop incremental features above databases such as sharding and data encryption. In addition, ShardingSphere allows developers not to worry about database configuration and shields user perception, so it quickly builds service-oriented database direct connection, which is well separated from the system architecture level, and ensures that daily maintenance work at the database agent, such as bug repair and version update, will not affect the services.\n\nThis time, ShardingSphere collaborated with the R&D team of E-hualu’s Hulu App, and helped them overcome data volume explosion for several times. Actually, this is just a tiny example of ShardingSphere’s various application scenarios around the world.\n\n### About E-hualu & Hulu App\n\nFounded in 2001, Beijing E-hualu Information Technology Co., Ltd. is a listed company under the state-owned enterprise China Hualu Group Co., LTD. directly administrated by the State-owned Assets Supervision and Administration Commission of the State Council (SASAC). Its mission is to reduce energy consumption and the cost of long-term data management.\n\nHulu App is the first to-customer product of E-hualu.\n\n### ShardingSphere Community:\n\nShardingSphere Github: https://github.com/apache/shardingsphere\n\nShardingSphere Twitter: https://twitter.com/ShardingSphere\n\nShardingSphere Slack Channel: apacheshardingsphere.slack.com","date":"2021-11-29","author":"SphereEx","excerpt":"ShardingSphere has greatly simplified sharding development and maintenance, which has played a very important role in our rapid product release. We estimate that ShardingSphere has saved us at least 4 months of R&D costs.","metaDescription":"Founded in 2001, Beijing E-hualu Information Technology Co., Ltd. is a listed company under the state-owned enterprise China Hualu Group Co., LTD. directly administrated by the State-owned Assets Supervision and Administration Commission of the State Council (SASAC). Its mission is to reduce energy consumption and the cost of long-term data management.","createdAt":"2021-12-12T12:33:14.398Z","updatedAt":"2022-09-26T10:27:50.013Z","publishedAt":"2021-12-12T12:35:01.820Z","locale":"en","caseStudiesType":{"data":{"id":1,"attributes":{"name":"Finance","createdAt":"2021-12-11T23:01:56.177Z","updatedAt":"2021-12-11T23:01:58.635Z","publishedAt":"2021-12-11T23:01:58.567Z","locale":"en"}}},"cover":{"data":{"id":7,"attributes":{"name":"ehualu.png","alternativeText":"ehualu.png","caption":"ehualu.png","width":1030,"height":720,"formats":{"thumbnail":{"name":"thumbnail_ehualu.png","hash":"thumbnail_ehualu_6d29db12b5","ext":".png","mime":"image/png","width":223,"height":156,"size":6.16,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_ehualu_6d29db12b5.png"},"large":{"name":"large_ehualu.png","hash":"large_ehualu_6d29db12b5","ext":".png","mime":"image/png","width":1000,"height":699,"size":53.63,"path":null,"url":"https://official-media.sphere-ex.com/large_ehualu_6d29db12b5.png"},"medium":{"name":"medium_ehualu.png","hash":"medium_ehualu_6d29db12b5","ext":".png","mime":"image/png","width":750,"height":524,"size":33.7,"path":null,"url":"https://official-media.sphere-ex.com/medium_ehualu_6d29db12b5.png"},"small":{"name":"small_ehualu.png","hash":"small_ehualu_6d29db12b5","ext":".png","mime":"image/png","width":500,"height":350,"size":18.22,"path":null,"url":"https://official-media.sphere-ex.com/small_ehualu_6d29db12b5.png"}},"hash":"ehualu_6d29db12b5","ext":".png","mime":"image/png","size":24.54,"url":"https://official-media.sphere-ex.com/ehualu_6d29db12b5.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2021-12-12T12:04:05.768Z","updatedAt":"2021-12-14T07:19:57.408Z"}}},"metaLogo":{"data":{"id":3,"attributes":{"name":"ehualu_logo.png","alternativeText":"ehualu_logo.png","caption":"ehualu_logo.png","width":303,"height":153,"formats":{"thumbnail":{"name":"thumbnail_ehualu_logo.png","hash":"thumbnail_ehualu_logo_26122a7add","ext":".png","mime":"image/png","width":245,"height":124,"size":22.6,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_ehualu_logo_26122a7add.png"}},"hash":"ehualu_logo_26122a7add","ext":".png","mime":"image/png","size":22.6,"url":"https://official-media.sphere-ex.com/ehualu_logo_26122a7add.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2021-12-12T12:04:05.036Z","updatedAt":"2021-12-14T07:19:30.734Z"}}},"localizations":{"data":[{"id":2,"attributes":{"feature":false,"title":"葫芦 App 后台数据处理的逻辑捷径","content":"> ShardingSphere 大大简化了分库分表的开发和维护工作，对于业务的快速上线起到了非常大的支撑作用，保守估计 ShardingSphere 至少为我们节省了 4 个月的研发成本。\n> \n> –– 史墨轩，易华录·技术总监\n\n### 项目背景\n\n伴随着易华录旗下面向个人用户的云服务产品【葫芦 App】正式上线，后台架构所承受的业务压力也与日俱增。\n\n为此，葫芦 App 研发团队选择采用 ShardingSphere 分库分表的功能对数据进行了横向的拆分，围绕 ShardingSphere 灵活敏捷的特性，满足了葫芦 App 业务对数据层扩展性的要求，避免团队重复“造轮子”，最大程度简化了随着业务增长而带来的愈发复杂化的分库分表的开发与维护工作。\n\n### 业务挑战\n\n- 扩展能力\n- 效率提升\n- 上线提前\n- 成熟技术\n\n### 解决方案\n\n- 可插拔架构的『高扩展性』\n\n由于业务特性，葫芦 App 原本有限的存储空间被消耗得非常快，并逐渐开始影响用户在前端的响应效率。通过采用 ShardingSphere 的分片策略，葫芦研发团队在应对海量计算+存储所带来的业务问题同时，能够确保分片扩展策略的灵活性。基于此，葫芦团队可以在ShardingSphere 上快速做出相应的功能扩展，为后续架构调整提供优化方案，进一步强化突出了 ShardingSphere 分库分表的优势。\n\n- 距离业务更紧密的『贴合性』\n\n后台架构的变化越小，对于业务而言就越可控。ShardingSphere 这种位于数据库之上的生态，距离业务更近，部署起来也更加轻量，无疑是解决葫芦 App 前台业务变更与后台架构调整之间矛盾的最优解。此外在灵活性层面，ShardingSphere 通过相关配置即可实现的方式，极大简化了葫芦研发团队在分库分表层面的研发与维护工作，对于业务的快速上线起到了非常大的支撑作用。\n\n- 对于业务架构的『零侵入性』\n\n葫芦 App 选择了 ShardingSphere-Proxy 部署模式，在不更换底层数据库的前提下通过 Proxy 来管理真实的数据库集群，基本无需对业务进行改造就已经完成了业务与数据在架构层面的分离，并避免了因更换数据库导致的业务不可用、漫长稳定周期等风险。另外，ShardingSphere 的无状态模式，几乎不会对前端用户产生任何可感知的影响，业务层也无需关注数据的存储方式。\n\n![ehualu_zh_1.png](https://official-media.sphere-ex.com/ehualu_zh_1_3fad8e2ccd.png)\n\n### 用户收益\n\n- 节省 4 人月成本\n- 明显改善业务体验\n- 系统灵活扩容\n\n**SphereEx，让数据与应用连接更简单。**","date":"2021-11-29","author":"SphereEx","excerpt":"ShardingSphere 大大简化了分库分表的开发和维护工作，对于业务的快速上线起到了非常大的支撑作用，保守估计 ShardingSphere 至少为我们节省了 4 个月的研发成本。","metaDescription":"北京易华录信息技术股份有限公司，成立于 2001 年，是国务院国资委直接监管的中央企业中国华录集团旗下控股的上市公司，致力于建设城市数字经济基础设施。","createdAt":"2021-12-11T23:11:13.132Z","updatedAt":"2022-11-30T08:25:32.894Z","publishedAt":"2021-12-11T23:11:16.643Z","locale":"zh"}}]}}},{"id":26,"attributes":{"feature":false,"title":"openGauss & ShardingSphere: A Top Distribution Solutions","content":"##About openGauss\n\nopenGauss is an open source relational database management system. Since its launch in June 2020, it has made waves the database management vertical. With an increasing number of partners and developers, it’s already reaping the benefits of the significant efforts & resources poured into building a successful database ecosystem.\n\n##Project Background\n\nFacing problems like data volume explosion, and super-high levels of concurrent data processing, openGauss made a choice: to adopt a distributed solution. It started to work on big data storage, throughput improvement for super-high concurrency, improving performance bottlenecks caused by large data amounts in a table, etc.\n\nApache ShardingSphere facilitates the distributed feature development of openGauss. Such collaboration on the full stack and open source solution helps openGauss make significant breakthroughs.\n\n##The Distributed Solution\n\nopenGauss integrates many open source components to optimize its full stack & open source distributed solution for horizontal scaling, distributed transactions, and distributed governance. Its framework is shown in Fig.1\n\n![Figure 1: The Structure of the ShardingSphere X openGauss Distribution Solution](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/s9knu2pwkmpsmvx6iwm2.png)\n\nShardingSphere-Proxy is the open source distributed database solution. \nShardingSphere’s well-known features include but are not limited to data sharding, distributed transaction, elastic scaling, and read-write splitting.\n\nHAProxy uses Patroni REST API to identify database master nodes and guarantee high availability, as well as load balancer.\n\nEvery high availability node of Patroni supports one master with multiple slaves and uses the Praxos protocols to keep data consistency, so that nodes are allowed to be deployed in the same site or in different sites to ensure multiple sites and centers data security.\n\nIn terms of the solution, ShardingSphere-Proxy contributes with its mighty distribution features, Kubernetes helps manage clusters, and prometheus monitors cluster status. This all-star team’s integration make the project a really powerful full stack, open source distribution solution.\n\n##What’s Special About the Project?\n\n + _Great extensibility and elastic scaling_\n\nHorizontal scaling helps realize the linear extension of computing and storage capabilities. The maximum shard count increases to 6400. Database performance improves accordingly and the problem of data volume explosion in single database is effectively managed.\n\nBased on users’ traffic condition, a user can flexibly scale data nodes, and adopt intelligent read-write splitting to realize automatic load balancing of a distributed database.\n\n+ _Enhanced enterprise-grade features_\n\nThe project provides users with various enterprise-grade features such as distributed data store, Trigger, distributed governance, comprehensive end-to-end encryption, and Workload Diagnosis Report (WDR)\n\n+ _Simple deployment_\n\nIt may take only a few seconds to deploy the project. Its deployment is simple while also being efficient as it adopts standardized image deployment to keep delivery consistent in different environments. It also uses container deployment as well as resource pooling so that it is less platform dependent.\n\n+ _Super-high availability and remote disaster recovery_\n\nThe project can effectively manage clusters and facilitate operations & maintenance. It follows the Paxos protocols to ensure data security and strong consistency and provides various features for disaster recovery, whose recovery point object (RPO) is zero. It supports resilient recovery methods for one region, cross-region, and multi-region & multi-center.\n\n+ _Open source ecosystem_\n\nThe open source project openGauss warmly welcomes more contributors, and encourages joint efforts to grow the open source database ecosystem.\n\n##Application Scenarios\n\n+ Financial system\n\nThe product is extremely reliable even for demanding financial systems. Paxos is a family of protocols for solving consensus in a network of unreliable and fallible processors. Besides being Paxos-based, the product includes powerful features such as distributed transaction with strong consistency, and multi-region multi-center disaster recovery whose RPO is 0.\n\n+ Government and enterprise operations\n\nIt’s really safe and convenient for government and enterprise operations due to its comprehensive end-to-end security protection, its hardware and software ecosystem support, and its high availability capabilities.\n\n+ Intelligent grid\n\nThe product’s flexible disaster recovery features can improve smart grid database scalability.\n\n##ShardingSphere Community:\n\n[ShardingSphere Github](https://github.com/apache/shardingsphere)\n\n[ShardingSphere Twitter] \n(https://twitter.com/ShardingSphere)\n\n[ShardingSphere Slack Channel](apacheshardingsphere.slack.com)\n\n\n\n\n\n","date":"2021-09-28","author":"SphereEx","excerpt":null,"metaDescription":"openGauss is an open source relational database management system that is released with the Mulan PSL v2. with the kernel built on Huawei’s years of experience in the database field and continuously provides competitive features tailored to enterprise-grade scenarios. In addition, openGauss is an open source database platform that encourages community contribution and collaboration.","createdAt":"2022-09-26T08:09:50.554Z","updatedAt":"2022-09-27T07:00:49.291Z","publishedAt":"2022-09-27T07:00:36.447Z","locale":"en","caseStudiesType":{"data":{"id":10,"attributes":{"name":"Software Development","createdAt":"2022-09-26T10:34:53.523Z","updatedAt":"2022-09-27T06:59:43.907Z","publishedAt":"2022-09-27T06:59:43.905Z","locale":"en"}}},"cover":{"data":{"id":346,"attributes":{"name":"openGauss1.png","alternativeText":"openGauss1.png","caption":"openGauss1.png","width":720,"height":411,"formats":{"thumbnail":{"name":"thumbnail_openGauss1.png","hash":"thumbnail_open_Gauss1_0781ec1284","ext":".png","mime":"image/png","width":245,"height":140,"size":31.7,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_open_Gauss1_0781ec1284.png"},"small":{"name":"small_openGauss1.png","hash":"small_open_Gauss1_0781ec1284","ext":".png","mime":"image/png","width":500,"height":285,"size":97.34,"path":null,"url":"https://official-media.sphere-ex.com/small_open_Gauss1_0781ec1284.png"}},"hash":"open_Gauss1_0781ec1284","ext":".png","mime":"image/png","size":124.92,"url":"https://official-media.sphere-ex.com/open_Gauss1_0781ec1284.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:07:47.594Z","updatedAt":"2022-09-26T08:07:47.594Z"}}},"metaLogo":{"data":{"id":361,"attributes":{"name":"openGauss logo.png","alternativeText":"openGauss logo.png","caption":"openGauss logo.png","width":275,"height":183,"formats":{"thumbnail":{"name":"thumbnail_openGauss logo.png","hash":"thumbnail_open_Gauss_logo_7b11da9186","ext":".png","mime":"image/png","width":234,"height":156,"size":11.4,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_open_Gauss_logo_7b11da9186.png"}},"hash":"open_Gauss_logo_7b11da9186","ext":".png","mime":"image/png","size":3.07,"url":"https://official-media.sphere-ex.com/open_Gauss_logo_7b11da9186.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T09:11:52.021Z","updatedAt":"2022-09-26T09:11:52.021Z"}}},"localizations":{"data":[]}}}],"meta":{"pagination":{"page":1,"pageSize":25,"pageCount":1,"total":17}}}},"caseStudiesFeatures":{"zh":[{"id":34,"attributes":{"feature":true,"title":"F6 汽车科技基于 Apache ShardingSphere 的核心业务分库分表实践","content":"## 业务背景\nF6 汽车科技，是一家专注于汽车后市场信息化建设的互联网平台公司，为维修企业开发智慧管理平台。各个维修企业（后文简称商户）之间的数据是相互隔离的，不同商户之间的数据理论上可以存储在不同库不同表。随着公司业务发展迅速，有些表的数据量增长迅速，单表总的数据量达到千万甚至亿的级别，系统越来越难以满足业务的高速发展。另外随着业务发展公司也在对系统进行拆分，按照不同域不同业务拆分成许多微服务，随之也就垂直拆分成了不同的业务库。  \n \n## 业务挑战\n关系型数据库由于单机存储容量、连接数、处理能力都有限，容易成为系统瓶颈。从性能上看，当单表的数据量达到千万以后，由于查询维度较多，即使添加从库、优化索引，做很多操作时性能仍下降严重。此时就要考虑对其进行切分了，切分的目的就在于减少数据库的负担，缩短查询时间。另外单库的连接数有限，如果数据库查询的 QPS 过高，那么就需要通过分库来分担单个数据库的连接压力。从可用性上看，单个数据库如果发生意外，很可能会丢失所有数据，影响所有业务，分库可以隔离故障，减少业务的影响范围。\n \n## 总体方案\n结合公司实际情况我们选择了 ShardingSphere 实现 Sharding。按照公司现有业务逻辑，因此采用商户 ID 作为分片键（ShardingKey），保证一个商户的工单数据，分配在同一个库同一个单表中，避免了多表查询关联的性能损耗，后期分片到多个库时，可以避免跨库事务及跨库 JOIN。\n商户 ID 数据库中的类型为 BIGINT(20)，为了保证后续分库扩容的需求，采用基因法，选取商户 ID 最后两位作为分库基因，按照双倍扩容的规则，可以最大扩展到 64 个库中。剩余位数的值用于分表，分片到 32 个分表中。\n\n规则如下（10545055917999668983 为某商户 ID）：\n\n```\n105450559179996689    83\n分表基因值 % 32    分库基因值 % 1\n```\n\n最后两位 83 用于分库，暂时数据只分片到 f6xxx 一个库中，因此余数为 0，后期数据量增加，扩展至多个库。剩余数值 105450559179996689 用于分表，首次分为 32 个单表，取模余数对应具体的分表下标，为 0~31。\n\n![screenshot-20230321-153539.png](https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/screenshot_20230321_153539_94108c0802.png)\n按照公司系统现状及开发功能能快速迭代分步上线的背景，因此我们制定了先分表再拆库的方案，考虑到分库分表改动影响比较大，因此需要灰度上线，一旦有问题需要快速回滚，不能影响正常业务。具体实施步骤如下：\n### 分表\n（1）工程从 JDBC 切到 Sharding-JDBC 数据源连接方式\n（2）写库业务解耦和代码迁移\n（3）历史数据和增量数据同步\n（4）切分表\n### 分库\n（1）读库业务迁移\n（2）数据迁移\n（3）切读库\n（4）切写库\n\n## 选择 Apache ShardingSphere 的理由\n- 社区活跃，目前已经升级至 5.0 版本，仍处于快速迭代中；\n- 成功应用案例多，京东、当当等大公司广泛应用；\n- 使用简单，Sharding-JDBC 快速集成项目，不需要部署额外服务；\n- 兼容性好，路由至单数据节点，SQL 100% 支持；\n- 性能优异，损耗低，官网有测试结果；\n\n\n","date":"2023-01-02","author":"SphereEx ","excerpt":"汽车互联网平台领域技术实践        ","metaDescription":null,"createdAt":"2023-03-21T07:36:34.241Z","updatedAt":"2024-01-03T09:57:02.591Z","publishedAt":"2023-03-21T07:36:35.817Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":28,"attributes":{"feature":true,"title":"中商惠⺠交易中台架构演进：对 Apache ShardingSphere 的应⽤","content":"> 项目背景\n中商惠民历经⼏次技术迭代，2016 到 2017 年开始 PHP 技术栈转到 Java 技术栈并进⾏微服务化，2018 年开始打造中台。\n2021 年年初完成的交易中台⼀期上线，2022 年 3 ⽉完成⼆期上线。\n\n伴随着业务成长和多样性的变化，要及时响应新需求，同时要避免对现有业务产⽣影响，本着提⾼⼈效，降低成本的原则，中商惠民推出了强中台发展战略。对系统⽽⾔，要降低系统之间的耦合度，增加可扩展性，提取业务共性的同时要保证低延迟。在 2021 年之初完成了对之前订单管理系统（OMS）的重构，推出了以订单的交易过程为核⼼的交易中台项⽬，分为以下四个核⼼模块：订单状态流转系统，订单管理系统，订单履约系统，订单费⽤计算系统。同时，基于业务分层，又拆分为订单查询系统，报表系统，订单管理后台等。\n\n在系统的重构过程中避免技术需求对业务需求产⽣阻碍，上线过程分为两个阶段：应⽤系统拆分和数据拆分。\n\n## 业务挑战\n数据表拆分之后，需要解决数据散列和查询问题。作为⼀家中⼩型企业，⾃研⼀套分库分表的中间件成本过⾼，初期还会⾯临各种踩坑的风险，\n\n## 拆分原则\n以 3 年作为⼀个迭代周期，拆分数量=（（每⽉新增的数据量* 36）+ 历史数据量）/单表数据量上限\n根据我们维护阿⾥云 RDS MySQL 的经验来看，基于在使⽤ Innodb 引擎下，进⾏ DDL 操作时，⼩于 500 万数据量，执⾏时间⼤概⼏⼗秒；⼤于 500 万，⼩于 1000 万数据量，执⾏时间⼤概 500 秒左右；⼤于 1000 万数据量，⼩于 5000 万左右，执⾏时间⼤概 1000 多秒；⼤于 5000 万以上数据量，执⾏时间在 2000 秒以上。\n单表分配上限可以取决于单表操作对业务带来的影响，当然也有⼀些⽅案可以解决 DDL 操作锁表的问题，例如双表同步，切换表名的⽅式可以将影响降低为秒级。\n库表的拆分数量最好是 2 的 N 次⽅，有利于后期做⽔平扩容。\n\n## 解决方案\nShardingSphere 提供了⼀种解决思路。它提出了连接模式（Connection Mode）的概念，将其划分为内存限制模式（MEMORY_STRICTLY）和连接限制模式（CONNECTION_STRICTLY）这两种类型。\n\n### 内存限制模式\n\n使⽤此模式的前提是，ShardingSphere 对⼀次操作所耗费的数据库连接数量不做限制。如果实际执⾏的 SQL 需要对某数据库实例中的 200 张表做操作，则对每张表创建⼀个新的数据库连接，并通过多线程的⽅式并发处理，以达成执⾏效率最⼤化。并且在 SQL 满⾜条件情况下，优先选择流式归并，以防⽌出现内存溢出或避免频繁垃圾回收情况。\n### 连接限制模式\n\n使⽤此模式的前提是，ShardingSphere 严格控制对⼀次操作所耗费的数据库连接数量。如果实际执⾏的 SQL 需要对某数据库实例中的 200 张表做操作，那么只会创建唯⼀的数据库连接，并对其 200 张表串⾏处理。如果⼀次操作中的分⽚散落在不同的数据库，仍然采⽤多线程处理对不同库的操作，但每个库的每次操作仍然只创建⼀个唯⼀的数据库连接。这样即可以防⽌对⼀次请求对数据库连接占⽤过多所带来的问题。该模式始终选择内存归并。\n\n内存限制模式适⽤于 OLAP 操作，可以通过放宽对数据库连接的限制提升系统吞吐量；连接限制模式适⽤于 OLTP 操作，OLTP 通常带有分⽚键，会路由到单⼀的分⽚，因此严格控制数据库连接，以保证在线系统数据库资源能够被更多的应⽤所使⽤，是明智的选择。\n\n我们发现在内存限制模式下，过程中会因为 MySQL 的 innodb 引擎的 cache buffer 加载策略导致操作变为 io 密集型，导致 SQL ⼤量超时，解决此问题的办法就是在不变更数据库资源的情况下我们程序多⼀层处理，如果发现没有分⽚键的时候，先在外置索引中确认⼀下分⽚键，再通过分⽚键来进⾏数据库检索。\n\n## 选择 Apache ShardingSphere 的理由\n\n1. Apache ShardingSphere 功能符合预期，可以解决⽬前的问题并且有丰富的扩展性；\n2. Apache ShardingSphere 社区活跃度⾼，遇到问题会有专⼈及时响应；\n3. 公司采⽤ SpringCloud 技术栈，集成⽅便，成本低；\n4. 性能表现符合预期，完全可以⽀撑现有业务。\n\n## 用户收益\n\n- 性能提升\n通过架构重构，有效控制单表数据量，⼤幅缩减慢 SQL，效率提升近 50%。\n\n- 节省研发资源，降低成本\n引⼊成熟的 Apache ShardingSphere ⽆需重新开发分表组件，降低了研发和踩坑的成本，研发同学只需要集中精⼒处理业务问题。\n\n- 丰富的扩展性\nApache ShardingSphere 对于数据加密、分布式事务和影⼦库等⽅⾯都具备良好的扩展性。","date":"2022-11-01","author":"SphereEx ","excerpt":"通过架构重构，有效控制单表数据量，⼤幅缩减慢 SQL，下降将近 50%","metaDescription":null,"createdAt":"2022-11-03T06:03:40.665Z","updatedAt":"2022-11-16T09:39:37.310Z","publishedAt":"2022-11-03T06:03:44.426Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":{"id":423,"attributes":{"name":"WechatIMG1962.png","alternativeText":"WechatIMG1962.png","caption":"WechatIMG1962.png","width":1790,"height":758,"formats":{"thumbnail":{"name":"thumbnail_WechatIMG1962.png","hash":"thumbnail_Wechat_IMG_1962_95b9c6004e","ext":".png","mime":"image/png","width":245,"height":104,"size":41.69,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/thumbnail_Wechat_IMG_1962_95b9c6004e.png"},"large":{"name":"large_WechatIMG1962.png","hash":"large_Wechat_IMG_1962_95b9c6004e","ext":".png","mime":"image/png","width":1000,"height":423,"size":461.64,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/large_Wechat_IMG_1962_95b9c6004e.png"},"medium":{"name":"medium_WechatIMG1962.png","hash":"medium_Wechat_IMG_1962_95b9c6004e","ext":".png","mime":"image/png","width":750,"height":318,"size":283.18,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/medium_Wechat_IMG_1962_95b9c6004e.png"},"small":{"name":"small_WechatIMG1962.png","hash":"small_Wechat_IMG_1962_95b9c6004e","ext":".png","mime":"image/png","width":500,"height":212,"size":140.24,"path":null,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/small_Wechat_IMG_1962_95b9c6004e.png"}},"hash":"Wechat_IMG_1962_95b9c6004e","ext":".png","mime":"image/png","size":966.17,"url":"https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/Wechat_IMG_1962_95b9c6004e.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-11-03T06:01:48.589Z","updatedAt":"2022-11-03T06:01:48.589Z"}}},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":29,"attributes":{"feature":true,"title":"Apache ShardingSphere 在转转亿级交易系统落地实践","content":"Apache ShardingSphere在转转亿级交易系统落地实践\n## 一、背景与挑战\n这几年随着转转二手业务的快速发展，订单系统的基础性能问题也愈发严重，作为系统运转的基石，订单库压力不容小觑。 面临的问题：\n\n* 大促期间DB压力大，单库查询qps上万占用大量数据库资源，写性能大大降低；\n\n* 数据与日剧增，单库中包含非常多数据量过数亿的大表，占用空间接近服务器的容量上限；\n* 数据量太大，数据备份和恢复需要耗费很长时间，极端情况下丢失数据的风险越高。\n\n\n## 二、为什么选ShardingSphere\n迫于上述所说的DB压力问题，起初我们做了一些缓解措施，其中包括:\n\n- 优化大事务，缩小事务范围，甚至消除事务\n\n![图片](https://mmbiz.qpic.cn/mmbiz_png/dHUzltsJpQvfq4yuZUZnaFySYwp5ImicXKkqdaSqd8r1g1hw53d3ps0u0lPrKAY39IrXE9EOcVsevianzHNUuqZA/640?wx_fmt=png&wxfrom=5&wx_lazy=1&wx_co=1)]\n\n通过调整原有业务逻辑顺序将核心的生单步骤放置在最后 ，仅在订单主表保持事务，主表操作异常时其他订单相关的表允许有脏数据产生。\n\n- 订单数据添加缓存\n\n![[图片]](https://mmbiz.qpic.cn/mmbiz_png/dHUzltsJpQvfq4yuZUZnaFySYwp5ImicXTvwRIHyN6TyERnd9vHloiasooia9bmfU2KdzKeJKrvZhu1BEaAeuqjicA/640?wx_fmt=png&wxfrom=5&wx_lazy=1&wx_co=1)\n\n缓存这块最重要同时最麻烦的地方是保证数据的一致性，订单信息涉及结算抽佣等，数据的不实时或不一致会造成严重事故。   严格保证缓存数据的一致性，代码实现比较复杂同时会降低系统的并发，因此缓存方案实现这块我们做了一定的妥协：1. 允许数据缓存失败情况下请求直接查库;2. 给缓存key添加版本号，通过读最新版本号的数据确保数据的实时性。\n\n- 复杂查询走ES、主从分离、一些大表进行冷热数据分离等\n\n![[图片]](https://mmbiz.qpic.cn/mmbiz_png/dHUzltsJpQvfq4yuZUZnaFySYwp5ImicXzz3vmTHe65ujHNRIWQvibnCwkre6Rg3ywArCBpqVJdibW1PfE5IgRcGQ/640?wx_fmt=png&wxfrom=5&wx_lazy=1&wx_co=1)\n\n通过上述几个方面的优化DB压力虽有所下降，但面对大促等一些高并发场景时仍显得力不从心。为了从根本上解决订单库的性能问题，我们决定对订单库进行数据分片（分库分表）改造，使得未来3-5年内不需要担心订单容量问题。\n数据分片组件选型这块，我们从效率、稳定性、学习成本和时间多个方面对比，最终选择了ShardingSphere。\n[图片]\n\n## ShardingSphere优势如下:\n\n- 提供标准化的数据分片、分布式事务和数据库治理功能，可适用于如Java同构、异构语言、容器、云原生等各种多样化的应用场景；\n- 分片策略灵活，支持多种分片方式；\n- 集成方便、业务侵入程度低；\n- 文档丰富、社区活跃。ShardingShpere提出DB Plus的理念，采用可插拔的架构设计，模块相互独立，可以单独使用，亦可以灵活组合使用。目前ShardingSphere 由 JDBC、Proxy 和 Sidecar（规划中）这 3 款既能够独立部署，又支持混合部署配合使用的产品组成。 \n\n3款产品特性对比如下：\n\n[![图片]](https://mmbiz.qpic.cn/mmbiz_png/dHUzltsJpQvfq4yuZUZnaFySYwp5ImicXjenibENKwjBdT4jC8093bFZ9fWnhL8Jd4ZeNcrJ9a4l964QSqww6OfQ/640?wx_fmt=png&wxfrom=5&wx_lazy=1&wx_co=1)\n\n通过上图对比，结合订单高并发特性，本次数据分片中间件选择了ShardingSphere-JDBC。   ShardingSphere-JDBC定位为轻量级 Java 框架，在JDBC层提供的额外服务。它使用客户端直连数据库，以 jar 包形式提供服务，无需额外部署和依赖，可理解为增强版的 JDBC 驱动，完全兼容 JDBC 和各种 ORM 框架。\n\n![[图片]](https://mmbiz.qpic.cn/mmbiz_png/dHUzltsJpQvfq4yuZUZnaFySYwp5ImicXMuiaTH16tr615K1HoVGQuXKYbicKrcQHFbUaWicAjLbicibc6nicQuZxGkYg/640?wx_fmt=png&wxfrom=5&wx_lazy=1&wx_co=1)\n\n随着5.x版本的发布，ShardingSphere还提供了许多新特性：\n\n1. 全新 distsql 用于加载及展示 shardingsphere配置信息\n2. 支持跨不同数据库实例的分片 join sql 查询\n3. 增加数据网关能力，支持异构数据库存储\n4. 支持在线动态创建及修改用户权限\n5. 新增自动化探针模块\n\n## 三、项目落地中的关键点\n### 分片key的选择\n当前订单id的生成，采用了时间戳+用户标识码+机器码+递增序列的方式，其中用户标识码取自买家id的第9~16位，该码是用户id生成时的真随机位，很适合作为分片key。\n\n- 选择该用户标识码作为分片key有如下优势：\n  - 可以使分到各个库表的数据尽可能均匀；\n  - 无论是以订单id、还是以买家id作为查询条件，都可以快速定位到具体分片位置；\n  - 同一买家的数据能分布到相同的库表，方便买家维度的聚合查询。\n \n![[图片]](https://mmbiz.qpic.cn/mmbiz_png/dHUzltsJpQvfq4yuZUZnaFySYwp5ImicX3xGKDAYaxibeGG7tnBUVQwKXCVVwbUZziaQ0S2MQ7dMnWquIguXarlpg/640?wx_fmt=png&wxfrom=5&wx_lazy=1&wx_co=1)\n\n具体分片策略上：我们采用了16库16表，用户标识码取模分库，用户标识码高四位取模分表。\n\n### 新老库数据迁移\n\n  - 迁移必须是在线的，不考虑停机迁移，迁移的过程中还会有数据的写入；\n  - 数据应该是完整的，C端体验是无感知的，迁移之后需要保证新库和旧库的数据是严格一致的；\n  - 迁移过程中需要做到可以回滚，一旦迁移过程中出现问题，可以立即回滚到源库，不会对系统可用性造成影响。\n  数据迁移步骤如下：双写->迁移历史数据->校验->老库下线。\n## 四、效果&收益\n\n- 解决了单库容量上限问题;\n- 数据分片后单库表的数据量大大减少，单表数据量由原来的近亿降为百万级别，总体性能大大提高;\n- 降低了因单库、单表数据过大极端情况数据丢失风险，减轻运维压力。\n以下是两次大促期间，下单服务接口调用量与接口耗时对比。\n\n改造前：\n\n![[图片]](https://mmbiz.qpic.cn/mmbiz_png/dHUzltsJpQvfq4yuZUZnaFySYwp5ImicXflshq7lNHV5g3mZPRz9tRcbzXODDaP9kCkU6v6KUrxQkFUJvFgINcQ/640?wx_fmt=png&wxfrom=5&wx_lazy=1&wx_co=1)\n\n改造后：\n\n![[图片]](https://mmbiz.qpic.cn/mmbiz_png/dHUzltsJpQvfq4yuZUZnaFySYwp5ImicXzUDnSOkNjMdRh9jDian4JciaU6dXXkWZv84pUHicFZx6sSsIm621pevlg/640?wx_fmt=png&wxfrom=5&wx_lazy=1&wx_co=1)\n\n## 五、总结感悟\n  任何公司的“分库分表项目”说白了，都不是一个考验点子思路的常见项目，更多的反而是对一个人、一个团队干活的细致程度、上下游部门的沟通协作、工程化的操作实施以及抗压能力的综合考验。同时业务的不断发展，又促使系统数据架构需要跟着不断升级，ShardingSphere以其良好的架构设计、高度灵活、可插拔和可扩展的能力简化了数据分片的开发难度，使研发团队只需关注业务本身，进而实现了数据架构的灵活扩展。","date":"2022-11-01","author":"张强，转转交易中台研发工程师","excerpt":"业务的不断发展，促使系统数据架构的不断升级","metaDescription":null,"createdAt":"2022-11-03T08:13:11.607Z","updatedAt":"2023-03-21T07:31:16.169Z","publishedAt":"2022-11-03T08:13:14.490Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":{"id":354,"attributes":{"name":"zhuanzhuan logo.png","alternativeText":"zhuanzhuan logo.png","caption":"zhuanzhuan logo.png","width":130,"height":130,"formats":null,"hash":"zhuanzhuan_logo_fd59d35610","ext":".png","mime":"image/png","size":1.85,"url":"https://official-media.sphere-ex.com/zhuanzhuan_logo_fd59d35610.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:45:59.332Z","updatedAt":"2022-09-26T08:45:59.332Z"}}},"metaLogo":{"data":{"id":354,"attributes":{"name":"zhuanzhuan logo.png","alternativeText":"zhuanzhuan logo.png","caption":"zhuanzhuan logo.png","width":130,"height":130,"formats":null,"hash":"zhuanzhuan_logo_fd59d35610","ext":".png","mime":"image/png","size":1.85,"url":"https://official-media.sphere-ex.com/zhuanzhuan_logo_fd59d35610.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:45:59.332Z","updatedAt":"2022-09-26T08:45:59.332Z"}}},"localizations":{"data":[]}}},{"id":30,"attributes":{"feature":true,"title":"基于 ShardingSphere 的得物数据库中间件平台“彩虹桥”演进之路","content":"## 前言\n随着得物 App 用户开始快速增长，业务线日趋丰富，也对底层数据库带来了较大的压力。各个业务线对于数据分片、读写分离、影子库路由等等的需求成为了刚需，所以需要一个统一的中间件来支撑这些需求，得物“彩虹桥”应运而生。\n\n在北欧神话中，彩虹桥是连结阿斯加德（ Asgard ）【1】和 米德加尔特（中庭/ Midgard ）的巨大彩虹桥。我们可以把它当作是“九界之间”的连接通道，也是进入阿斯加德的唯一稳定入口。而得物的彩虹桥是连接服务与数据库之间的数据库中间层处理中间件，可以说得物的每一笔订单都与它息息相关。\n\n## 1. 技术选型\n\n前期我们调研了 Mycat、ShardingSphere、kingshard、Atlas 等开源中间件，综合了适用性、优缺点、产品口碑、社区活跃度、实战案例、扩展性等多个方面，最终我们选择了 ShardingSphere 。准备在 ShardingSphere 的基础上进行二次开发、定制一套适合得物内部环境的数据库中间件。\n\nApache ShardingSphere 是一款开源分布式数据库生态项目，由 JDBC、Proxy 和 Sidecar（规划中） 3 款产品组成。其核心采用可插拔架构，通过组件扩展功能。对上以数据库协议及 SQL 方式提供诸多增强功能，包括数据分片、访问路由、数据安全等；对下原生支持 MySQL、PostgreSQL、SQL Server、Oracle 等多种数据存储引擎。ShardingSphere 已于2020年4月16日成为 Apache 软件基金会的顶级项目，并且在全球多个国家都有团队在使用。\n\n目前我们主要是 ShardingSphere的Proxy 模式提供服务，后续将会在 JDBC&Proxy 混合架构继续探索。\n\n## 2.彩虹桥目前的能力\n\n![](https://oscimg.oschina.net/oscnet/up-8589dc26cf4195958a56f1878375ad2ab62.png)\n\n其中白色模块为现阶段以及具备的能力，绿色模块为规划&正在做的功能。下面介绍一下几个重点功能。注意，以下功能都是基于 Proxy 模式。\n\n### 2.1 数据分片\n\n数据分片指按照某个维度将存放在单一数据库中的数据分散地存放至多个数据库或表中以达到提升性能瓶颈以及可用性的效果。数据分片的有效手段是对关系型数据库进行分库和分表。分库和分表均可以有效地避免由数据量超过可承受阈值而产生的查询瓶颈。除此之外，分库还能够用于有效地分散对数据库单点的访问量；分表虽然无法缓解数据库压力，但却能够提供尽量将分布式事务转化为本地事务的可能，一旦涉及到跨库的更新操作，分布式事务往往会使问题变得复杂。使用多主多从的分片方式，可以有效地避免数据单点，从而提升数据架构的可用性。\n\n通过分库和分表进行数据的拆分来使得各个表的数据量保持在阈值以下，以及对流量进行疏导应对高访问量，是应对高并发和海量数据系统的有效手段。数据分片的拆分方式又分为垂直分片和水平分片。\n\n按照业务拆分的方式称为垂直分片，又称为纵向拆分，它的核心理念是专库专用。彩虹桥主要提供的分片能力是水平分片，水平分片又称为横向拆分。相对于垂直分片，它不再将数据根据业务逻辑分类，而是通过某个字段（或某几个字段），根据某种规则将数据分散至多个库或表中，每个分片仅包含数据的一部分。例如：根据主键分片，偶数主键的记录放入 0 库（或表），奇数主键的记录放入 1 库（或表），如下图所示。\n\n![](https://oscimg.oschina.net/oscnet/up-03995f1fc41e54091d476d67a3a3146a382.png)\n\n水平分片从理论上突破了单机数据量处理的瓶颈，并且扩展相对自由，是数据分片的标准解决方案。\n\n当然实际使用场景的分片规则是非常复杂的，我们提供一些内置算法比如取模、HASH 取模、自动时间段分片算法、Inline 表达式等。当内置算法无法满足要求时，还可以基于 groovy 来定制专属的分片逻辑。\n\n![](https://oscimg.oschina.net/oscnet/up-6d53404d8c0b8a4373c971f612169a5bd11.png)\n\n### 2.2 读写分离\n\n面对日益增加的系统访问量，数据库的吞吐量面临着巨大瓶颈。对于同一时刻有大量并发读操作和较少写操作类型的应用系统来说，将数据库拆分为主库和从库，主库负责处理事务性的增删改操作，从库负责处理查询操作，能够有效的避免由数据更新导致的行锁，使得整个系统的查询性能得到极大的改善。通过一主多从的配置方式，可以将查询请求均匀地分散到多个数据副本，能够进一步地提升系统的处理能力。\n\n与将数据根据分片键打散至各个数据节点的水平分片不同，读写分离则是根据 SQL 语义的分析，将读操作和写操作分别路由至主库与从库。\n\n![](https://oscimg.oschina.net/oscnet/up-3afd2cc77cd8369c2bb82b177f916342753.png)\n\n这里配置的方式比较简单，给目标主库绑定一个或多个从库、设置对应的负载均衡算法即可。\n\n![](https://oscimg.oschina.net/oscnet/up-5b17dae8d16b16886913af6bde92c238c27.png)\n\n这里的实现方式就是通过 SQL 解析，把查询语句路由到对应的从库即可，但是在一些对主从同步延迟比较敏感的场景，可能需要强制走主库，这里我们也提供一个 API（原理就是 SQL Hint），让上游可以指定某些模块读强制走主，还有相关全局配置可以让事务内所有读请求全部走主。\n\n### 2.3 影子库压测\n\n在基于微服务的分布式应用架构下，业务需要多个服务是通过一系列的服务、中间件的调用来完成，所以单个服务的压力测试已无法代表真实场景。在测试环境中，如果重新搭建一整套与生产环境类似的压测环境，成本过高，并且往往无法模拟线上环境的复杂度以及流量。因此，业内通常选择全链路压测的方式，即在生产环境进行压测，这样所获得的测试结果能够准确地反应系统真实容量和性能水平。\n\n全链路压测是一项复杂而庞大的工作。需要各个微服务、中间件之间配合与调整，以应对不同流量以及压测标识的透传。通常会搭建一整套压测平台以适用不同测试计划。在数据库层面需要做好数据隔离，为了保证生产数据的可靠性与完整性，需要将压测产生的数据路由到压测环境数据库，防止压测数据对生产数据库中真实数据造成污染。这就要求业务应用在执行 SQL 前，能够根据透传的压测标识，做好数据分类，将相应的 SQL 路由到与之对应的数据源。\n\n这里配置的方式类似读写分离，也是给目标主库绑定一个影子库，当 SQL 携带了影子标就会被路由到影子库。\n\n![](https://oscimg.oschina.net/oscnet/up-043144bd50a5daee6ff5781f3cf5236f6f5.png)\n\n### 2.4 限流&熔断\n\n当 DB 的压力超过自身水位线时，会导致 DB 发生故障。当我们预估出某个维度的水位线后，可以配置对于的限流规则，拒绝掉超过本身水位线以外的请求来保护 DB。让系统尽可能跑在最大吞吐量的同时保证系统整体的稳定性。维度方面我们支持 DB、Table、SQL 以及 DML 类型。\n\n![](https://oscimg.oschina.net/oscnet/up-0f50d0618386e4678c772a164ca35091bf3.png)\n\n彩虹桥下面连接了上百个 RDS 实例，每个 RDS 实例都有可能出现各种故障，当单个实例出现故障会影响到整个逻辑库，会迅速造成阻塞诱发雪崩效应。所以我们需要一种快速失败的机制来防止雪崩，目前我们是支持 DB 实例级别的熔断，基于获取连接& SQL 执行 2 种行为，以及执行时间跟失败比例来实现熔断，以达到在 DB 故障时快速失败的效果。\n\n![](https://oscimg.oschina.net/oscnet/up-916a3c6015f00af9c677499da18fc67558e.png)\n\n### 2.5 流量纠偏\n\n在双活架构下，彩虹桥作为数据库的代理层，可以保证双活架构下流量切换过程的流量做兜底拦截，保证数据一致性。原理就是基于 SQL Hint 携带的 userId 与机房规则做匹配，拦截不属于当前机房的流量。\n\n## 3. 基于 ShardingSphere 我们做了哪些改造\n\n虽然 ShardingSphere Proxy 本身其实已经足够强大，但是针对得物内部环境，还是存在一些缺陷和功能缺失，主要分为以下几点：\n\n* 易用性\n\n    * 分片、读写分离等规则配置文件过于复杂，对于业务开发不够友好\n\n    * 规则动态变更完全依赖配置中心，缺失完善的变更流程\n\n    * 连接池治理能力不完善\n\n    * Hint 方式不够友好，需要业务写 RAL 语句\n\n    * 自定义分片算法需要发布\n\n    * SQL 兼容性\n\n* 稳定性\n\n    * 多集群治理功能缺失\n    \n    * 逻辑库之间的隔离性缺失\n     \n    * 限流熔断组件缺失\n     \n    * 数据源、规则动态变更有损\n     \n    * 双活架构下流量纠偏功能的缺失\n     \n    * 发布有损\n\n* 可观测性\n \n    * SQL Trace 功能不完善\n     \n    * 监控指标不够全面\n     \n    * SQL 洞察能力缺失\n\n* 性能\n\n    * 由于多了一次网络转发，单条 SQL 的 RT 比直连会上浮 2~3ms\n\n为了解决以上问题，我们做了以下改造与优化。\n\n### 3.1 易用性提升\n\n* 针对数据源、规则的配置、变更、审计等一系列操作，集成到管控台进行统一操作。通过图形化的方式降低了分片、读写分离等规则配置文件的复杂度，并且加上一系列校验来规避了一部分因为配置文件错误导致的低级错误。其次加上了审计功能，保障了配置动态变更的安全性和可控性。\n\n* 管控台新增了连接池治理，基于 RDS 连接数、Proxy 节点数、挂载数据源数量等因素自动计算出一个安全合理的连接池大小。\n\n* 新增 Client，针对 Hint 做一系列适配，比如影子标传递、双活架构用户 id 传递、trace 传递、强制路由等等。使用方只需要调用 Client 中的 API，即可在发出 SQL 阶段自动改写成 Proxy 可以识别的 Hint 增强语句。\n\n* 管控台新增了集群治理功能：由于我们部署了多套 Proxy 集群，为了最大程度的保障故障时的爆炸半径，我们按照业务域对 Proxy 集群进行了划分，尽量保证统一业务域下面的逻辑库流量进入同一套集群。\n\n* 新增了 Groovy 来支持自定义分片算法，在后台配置好分片逻辑后审核通过即可生效，无需 Proxy 发版\n\n### 3.2 稳定性提升\n\n#### 3.2.1 Proxy 多集群治理\n\n（1）背景\n\n随着 Proxy 承载的业务域越来越多，如果所有的流量都通过负载均衡路由到Proxy 节点，当一个库出现问题时，可能会导致整个集群瘫痪，爆炸范围不可控。而且由于 DB 的连接数资源有限，这就导致 Proxy 的节点无法大规模横向扩展。\n\n（2）解决方案\n\n为了不同业务域之间的隔离性，我们部署了多套 Proxy 集群，并且通过管控台维护各个逻辑库与集群之间的关系。保证同一业务域下面的逻辑库流量进入同一套集群。并且在某个集群发生故障时，可把故障集群中的逻辑库迅速、无损的动态切换至备用集群，尽可能减少 Proxy 本身故障给业务带来的损失。\n\n而针对连接数治理方面，Proxy 在初始化连接池的时候会判断一下当前逻辑库是否在当前集群，如果不在把最小连接数配置设置成最小，如果在则按照正常配置加载，并在集群切换前后做好目标集群的预热以及原集群的回收。这样可以极大程度上缓解 DB 连接池资源对 Proxy 节点的横向扩展。\n\n（3）实现原理\n\n上游应用引入 Rainbow（自研连接池）后，在连接池初始化之前会根据逻辑库读取当前库所在集群，并动态把 Proxy 的域名替换成其所在集群的域名。同时还会新增对集群配置的监听，这样在管控台切换集群操作后，Rainbow 会根据切换后的集群域名创建一个新的连接池，然后替换掉老的连接池，老的连接池也会进行延时优雅关闭，整个过程对上游应用无感知。\n\n（4）架构图\n\n![](https://oscimg.oschina.net/oscnet/up-79cf73883082b7b4f2a710c21ddee45ac22.png)\n\n#### 3.2.2 Proxy 工作线程池隔离\n\n（1）背景\n\n开源版本的 Proxy，所有逻辑库共用一个线程池，当单个逻辑库发生阻塞的情况，会耗尽线程池资源，导致其他逻辑库也跟着受影响。\n\n（2）解决方案\n\n我们这里采用了基于线程池的隔离方案，引入了独占&共享线程池的概念，优先使用逻辑库独占线程池，当独占线程池出现排队情况再使用共享线程池，共享线程池达到一定负载后会在一个时间窗口内强制路由到独占线程池，在保障隔离性的前提下，最大化利用线程资源。\n\n#### 3.2.3 流控与熔断\n\n（1）背景\n\n开源版本的 Proxy缺少对库、表、SQL 等维度的流控，当短时间内爆发超过系统水位的流量时，很可能就会击垮 DB 导致线上故障，而且缺少针对 DB 快速失败的机制，当 DB 发生故障（比如 CPU 100%）无法快速失败的情况下，会迅速造成阻塞诱发雪崩效应。\n\n（2）解决方案\n\n新增了各个维度（DB、table、SQL、语句类型）的限流，各个库可以根据预估的水位线以及业务需要配置一个合理的阈值，最大程度保护 DB 。同时我们也引入 DB 实例的熔断策略，在某个实例出现故障时可快速失败。在分库场景下，可最大程度减少对其他分片的影响，进一步缩小了故障的爆炸半径，提升彩虹桥整体的稳定性。\n\n（3）实现原理\n\n流控跟熔断都是基于 sentinel 来实现的，在管控台配置对应的规则即可。\n\n#### 3.2.4 无损发布\n\n（1）背景\n\n在前期，Proxy 每次发布或者重启的时候，都会收到上游应用 SQL 执行失败的一些报警，主要的原因是因为上游与 Proxy 之间是长连接，这时如果有连接正在执行 SQL，那么就会被强制断开导致 SQL 执行失败，一定程度上对上游的业务造成损失。\n\n（2）解决方案\n\n发布系统配合自研连接池 Rainbow，在 Proxy 节点发布或重启之前，会通知 Rainbow 连接池优雅关闭应用于需要重启&发布的 Proxy 节点之间的连接，在 Proxy 流量跌 0 后再执行重启&发布。\n\n### 3.3 可观测性\n#### 3.3.1 运行时指标\n\n（1）背景\n\n开源版本对于 Proxy 运行时的监控指标很少，导致无法观测到 Proxy 上面每个库运行的详细状态。\n\n（2）解决方案\n\n在 Proxy 各个执行阶段加了埋点，新增了以下指标、并绘制了对应的监控大盘。\n\n* 库&表级别的 QPS、RT、error、慢 SQL 指标\n \n* Proxy-DB 连接池的各项连接数指标\n \n* 流控熔断指标\n \n* 线程池活跃线程数、队列大小指标\n\n（3）效果图\n\n![](https://oscimg.oschina.net/oscnet/up-c7bb9bdd9ab8438333c1c801bbbf485c02e.png)\n\n![](https://oscimg.oschina.net/oscnet/up-63f959dba22c7f0ffb5646aa08da29708a7.png)\n\n![](https://oscimg.oschina.net/oscnet/up-b7d5d5e4c1595ace1819fabe3a29eb9bb2f.png)\n\n#### 3.3.2 全链路追踪\n\n（1）背景\n\n开源版本的 trace 只支持 Proxy 内部执行阶段的链路追踪，无法和上游串联，导致排障效率低下。\n\n（2）解决方案\n\n主要通过 RAL、SQL 注释 2 种方式传递 trace 信息，实现了上游到 Proxy 的全链路追踪。\n\n（3）实现原理\n\n我们首先想到的方案就是通过 SQL 注释方式传递 trace 信息，但是在真正投产之后却发现了一些问题，在上游使用了 prepare 模式（useServerPrepStmts=true&cachePrepStmts=true）时，Proxy 会缓存statmentId 和 SQL ，而 trace 每次都不一样，这样会导致存储缓存无限增长最终导致 OOM 。\n\n![](https://oscimg.oschina.net/oscnet/up-1fad2c4ac07b66b974efdf6f5886ff3abbd.png)\n\n所以通过 SQL 注释方式传递 trace 信息只适用于非 prepare 场景。于是我们又新增了一种方案就是在每次 SQL 执行之前发送一条 RAL 语句来传递 trace 信息，并在 Proxy 中缓存 channelId 与 trace 信息的对应关系，由于单个channel所有的SQL都是串行执行，加上 channelId 数量可控，不会无限膨胀。但是这种方案相对于每次 SQ L执行之前都有一次 RAL 语句的执行，对性能的影响还是比较大的。从监控上看下来每次 SQL 执行的 RT 会上浮 2-3ms（网络传输），对于一些链路较长的接口来说还是挺致命的。\n\n（4）总结\n\n综合下来 2 种方案其实都有比较明显的缺点，针对这个问题之前 ShardingSphere 的小伙伴来过我们得物进行过一次深度沟通，亮哥给出了一个比较可行的方案，就是通过虚拟列传输trace信息，但是需要上游对 SQL 进行改写，这可能会增加上游应用的负担，目前这块我们还没有开始做。\n\n#### 3.3.3 SQL 洞察\n\n（1）背景\n\n目前 Proxy 虽然有打印逻辑 SQL 和物理 SQL 的日志，但是由于生产请求量较大，开启日志会对 IO 有较大挑战。所以目前我们生产环境还是关闭的状态。而且这个日志也无法与上游串联，对于排障的帮助也是比较有限。\n\n（2）解决方案\n\n目前也只有个大概的思路，还没有完善的方案，要达到的效果就是收集所有 Proxy 执行的逻辑 SQL、物理 SQL 以及上游信息，包括 JDBCDatabaseCommunicationEngine 以外的一些 SQL（比如 TCL 等等），并通过管控台实时查询。最终的效果类似阿里云 RDS 的收费服务-SQL 洞察\n\n### 3.4 bug 修复\n\n由于历史原因，我们是基于 Apache ShardingSphere 5.0.0-alpha 上做的二次开发，在实际使用的过程中遇到了很多 bug，大部分都给官方提了 issue，并且在彩虹桥版本上做了修复，当然 ShardingSphere 社区的小伙伴也给与了很多帮助和修复的思路。\n\n### 3.5 JDBC&Proxy 混合架构\n\n（1）背景\n\n上面 4 项内容大多数针对 Proxy 模块或上游连接池模块的改造，但是还是有一些问题是 Proxy 模式暂时无法解决的，比如前面提到的 SQL 兼容性和性能问题。还有就是如果整个 Proxy 集群全部宕机情况下我们没有一个兜底机制。所以 Proxy 模式并不适用于所有场景。在 Apache ShardingSphere 的官方文档可以看到这样一段内容：\n\n![](https://oscimg.oschina.net/oscnet/up-2e0d1f8836d71db1cea63aa5b1cda5f1057.png)\n\n于是我们准备在 JDBC&Proxy 混合架构上做了进一步探索。\n\n（2）解决方案\n\n在管控台实现对逻辑库的模式配置，并通过自研连接池 Rainbow 感知并根据不同模式启动不同类型的数据源。并且可以在后台切换模式后无损动态调整，这样对于使用者来说是完全无感知的。对 SQL 性能、兼容性要求比较高的应用可以调整为 JDBC 模式。同时当 Proxy 所有集群瘫痪时也有个兜底的方案，不至于全站崩溃。\n\n（3）实现原理\n\nRainbow 连接池启动的时候会查询当前逻辑库对应的模式，如果是 Proxy 模式则直接连接 Proxy 来启动连接池，如果是 JBDC 则根据该逻辑库的数据源配置以及分片&读写分离&影子库等等规则来加载 JDBC 模式的数据源，对应的 DataSource 为 GovernanceShardingSphereDataSource。并且会监听这个模式配置，当模式发生变化后会动态无损替换当前连接池。具体的无损替换方案与前面提到的集群切换类似。同时需要解决的还有监控问题和连接池资源的管理，Proxy 切换至 JDBC 模式后指标的暴露由 Proxy 节点变成了上游节点，对应的大盘也需要做对应的融合，连接池治理这块也使用了新的计算模式做了对应的适配。\n\n## 4. 目前的困惑\n\n由于历史原因，我们是基于 Apache ShardingSphere 5.0.0-alpha 上做的二次开发，目前社区最新的版本是 5.1.2-release，从 5.0.0-alpha ~ 5.1.2-release 做了大量的优化跟 bug 修复，但是我们没有很好的办法将社区的代码合并到我们的内部代码，因为无法确定社区开源版本更改的内容，会不会对现有业务产生影响，短时间内也无法享受社区带来的红利。同时我们也在寻找一种方式将我们做的一些优化后续合并到社区中，也算是一种反哺社区。为中国开源做一份贡献~","date":"2022-11-01","author":"陈浩","excerpt":"得物的彩虹桥是连接服务与数据库之间的数据库中间层处理中间件，可以说得物的每一笔订单都与它息息相关。","metaDescription":null,"createdAt":"2022-11-03T09:00:48.784Z","updatedAt":"2022-11-03T09:00:51.175Z","publishedAt":"2022-11-03T09:00:51.172Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":31,"attributes":{"feature":true,"title":"ShardingSphere 在数十个联通政务热线场景中的应用：稳定、高效、可复制","content":"联通数科整合中国联通“云、大、物、智、链、安”（云计算、大数据、物联网、人工智能、区块链、安全）等能力，面向智慧城市、数字政府、工业互联网、生态环保、文化旅游等领域，联通数科已积累大量行业标杆案例，成功为客户提供更加丰富、专业的产品和服务。\n\n## 一、项目背景\n\n随着“互联网+政务服务”政策的持续发力，越来越多的地方政府开始着手打造“互联网+”环境下的政务服务平台。\n\n近年来，联通数科先后助力数十个政务热线平台上线，以提升政务服务能力，部分项目如下图。\n\n![up-20ec9e57d65c64dcf55a4277105b5ff0488.webp](https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/up_20ec9e57d65c64dcf55a4277105b5ff0488_6143b4132c.webp)\n\n政务热线业务有并发高、数据量大的特点，我们每拨打一次政务热线，即产生一条工单记录。相比以往，疫情期间的热线业务量更是翻了数倍。\n\n**在政务热线的工单模块中，为满足大规模、海量数据以及高稳定的业务需求，联通数科采用 ShardingSphere 对工单信息进行分片存储，在政民互动的桥梁中构建坚固的桥墩**。\n\n## 二、业务挑战\n\n政务热线是政府与企业、群众互动的主要渠道，以单一号码面对公众，提供 7 * 24 服务。除受理工单外，还包括了话务、知识库、语音对答及智能坐席等业务。\n\n在系统规划阶段，数据库层作为业务系统的基石，技术选型过程更为谨慎。系统的稳定是硬性指标，其次是性能，性能直接影响热线业务的服务能力，同时还要具备易于维护和管理的特点，便于系统升级及后台维护。\n\n在数据库架构选型中，政务热线系统对架构要求如下：\n\n-   成熟稳定\n    \n-   高性能\n    \n-   易维护\n    \n-   业务代码低耦合\n    \n\n## 三、选择 ShardingSphere 的理由\n\n**经过技术团队多轮调研测试，包括稳定性、产品功能、接入方式及性能等，最终选择“ShardingSphere + MySQL”分布式数据库解决方案**。\n\n-   理念先进\n    \n\n随业务应用场景多元化发展，数据库已呈碎片化趋势，「**Database Plus**」旨在数据库上层建立标准和生态，为数据库提供增量能力，可防止数据库绑定、实现自主可控，突破原有数据库功能局限，提升数据基础设施整体功能上限。\n\n-   成熟稳定/案例多\n    \n\nShardingSphere 于 2016 年开源，经过多年的研发迭代，产品历经多个头部互联网大型场景打磨，成熟稳定、值得信赖。\n\n-   功能全面\n    \n\nShardingSphere 除了具备 Sharding 能力外，数据加解密和影子库的功能也是作为联通技术团队评估的指标，通过一套技术栈既可解决安全和压测的场景需求。ShardingSphere 为政务热线系统建设及架构扩展，提供了更多的能力支持。\n\n-   高兼容/接入方便\n    \n\nShardingSphere 与 MySQL 协议兼容，语法支持度也非常高，在开发过程中几乎不用考虑 SQL 的支持问题，接入非常方便。\n\n-   高性能\n    \n\n政务热线忙时有数千坐席在线，工单记录以千万记录为起始单位，对性能有明确的要求。ShardingSphere-JDBC 定位为轻量级的 Java 框架，压测结果数据理想，可满足政务系统服务能力要求。\n\n-   易运维\n    \n\n在 JDBC 良好的性能基础之上，ShardingSphere 还提供了对运维人员友好的 Proxy，使用常用的客户端即可直接访问，上手维护较容易。\n\n除了以上五个维度的权衡，选型中也参考了其他数据库中间件产品，以下表格是 ShardingSphere 和 Mycat 的比对。\n\n|   | Apache ShardingSphere | Mycat |\n| --- | --- | --- |\n| 开源协议 | Apache-2.0 | GPL-2.0/GPL-3.0 |\n| 接入端 | 客户端/代理端 | 代理端 |\n| 数据库支持 | 客户端：任意  \n代理端：MySQL/PG/openGauss | MySQL、Oracle、DB2、SQL Server、PostgreSQL 等 |\n| 性能 | 客户端：损耗低  \n代理端：损耗高 | 损耗高 |\n| 读写分离 | 有 | 有 |\n| 数据加密 | 有 | 无 |\n| 全链路压测 | 有 | 无 |\n| 社区成熟度 | Apache 软件基金会  \n信通院可信认证  \n社区活跃度高 | 个人/三方社区  \n社区活跃度一般 |\n| 商业公司支持 | 有 | 无 |\n\n## 四、解决方案\n\n由于“ShardingSphere+MySQL ”具备稳定、易用及性能极致等特点，该方案在联通数科的多个政务热线项目中复制使用。\n\n-   部署方式\n    \n\nShardingSphere-JDBC 和 ShardingSphere-Proxy 混合使用的方式，兼顾了系统性能和可维护性。\n\n-   分片键/分片算法\n    \n\n政务热线的工单表相对较大，且使用逻辑简单，因此在分片设计上，选取工单 ID 作为分片键，通过哈希算法分布数据。\n\n-   迁移\n    \n\n新建项目不涉及历史数据，因此无需考虑数据迁移环节，上线即直接使用。\n\n-   实施过程\n    \n\n回忆数个已上线的项目，由于技术方案评估较全面，且多次复制方案，实施过程并没有遇到“坑”。\n\n-   系统架构\n    \n\n政务热线业务模块采用了微服务架构，节点在十几到几十不等，以具体省市规模决定。在数据库层，使用物理机部署 3 节点 MHA，每个政务热线系统一般需要 4 套 MHA 来支撑业务。\n\n![up-6eb6faa996c81e6de688a2dd9e204f730e6.webp](https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/up_6eb6faa996c81e6de688a2dd9e204f730e6_e867e826e4.webp)\n\n## 五、用户收益\n\n-   业务角度\n    \n    -   支持客户端模式，最大化保证了政务热线**对性能的要求**；\n        \n    -   **系统稳定**，为业务连续性带来有力的支撑。\n        \n-   研发角度\n    \n    -   **兼容良好**，兼容主流数据库 SQL 语法，对接即可使用；\n        \n    -   **功能全面**，读写分离、数据加密及全链路压测方案，系统扩展性强。\n        \n-   运维角度\n    \n    -   支持代理端模式，为运维人员提供了**良好的维护入口**。\n        \n-   技术选型角度\n    \n    -   **防止数据库绑定**，为未来升级提供了足够的灵活度支撑。\n        \n\n## 六、总结\n\n通过联通数科政务热线的多个案例，验证了 ShardingSphere 在政务场景的能力，也进一步说明了 ShardingSphere 无行业限制的特点。\n\n目前我国数字政府建设已进入全面提速阶段，联通数科和 ShardingSphere 通力合作，希望助力更多政务热线项目落地，打造更多高效、便捷和安全的政务热线。\n\n","date":"2022-11-01","author":"SphereEx 伙伴","excerpt":"通过联通数科政务热线的多个案例，验证了 ShardingSphere 在政务场景的能力，也进一步说明了 ShardingSphere 无行业限制的特点。","metaDescription":null,"createdAt":"2022-11-03T09:06:14.351Z","updatedAt":"2023-08-24T03:32:25.633Z","publishedAt":"2022-11-03T09:06:16.445Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":32,"attributes":{"feature":true,"title":" 数据为王，携程国际火车票的 ShardingSphere 之路","content":"以下文章来源于携程技术，作者瑞华\n\n**作者简介**\n\n瑞华，携程高级后端开发工程师，关注系统架构、分库分表、微服务、高可用等。\n\n## **一、前言**\n\n随着国际火车票业务的高速发展，订单量快速增长，单数据库瓶颈层面的问题逐渐显露，常规的数据库优化已无法达到期望的效果。同时，原先的底层数据库设计，也存在一些历史遗留问题，比如存在部分无用字段、表通过自增主键关联和各个应用直连数据库等问题。\n\n为此，经过讨论后，我们决定对订单库进行分库分表，同时对订单表进行重构，进而从根本上解决这些问题。\n\n## **二、问题挑战**\n\n目标确定后，实践起来可不轻松，出现了很多的问题和挑战。这里列举一些典型问题，大致可以分为两大类：分库分表通用问题、具体业务关联问题。\n\n**分库分表通用问题**\n\n-   如何切分，垂直分还是水平分？分片的键，如何选取？\n    \n-   如何根据键值路由到对应库、对应表？\n    \n-   采用什么中间件，代理方式还是中间件的方式？\n    \n-   跨库操作等问题，如跨库事务和跨库关联？\n    \n-   数据扩容问题，后续如何进行扩容？\n    \n\n**具体业务关联问题**\n\n-   各个应用直连数据如何解决？\n    \n-   如何进行平滑过渡？\n    \n-   历史数据如何恰当迁移？\n    \n\n## **三、方案选型**\n\n### **3.1 如何切分**\n\n切分方式，一般分为垂直分库、垂直分表、水平分库和水平分表四种，如何选择，一般是根据自己的业务需求决定。\n\n我们的目标是要从根本上解决数据量大、单机性能问题等问题，垂直方式并不能满足需求，所以我们选取了水平分库+水平分表的切分方式。\n\n### **3.2 分片键选取**\n\n一般是根据自己的实际业务，来选择字段来作为分片的键，同时可以结合考虑数据的热点问题 、分布问题。比如订单系统，不能根据国家字段进行分片，否则可能会出现某些国家很多的订单记录，某些国家几乎没有订单记录，进而数据分布不均。相对正确的方式，比如订单类系统，可以选择订单 ID；会员系统，可以选择会员 ID。\n\n### **3.3 如何路由**\n\n选定了分片的键之后，接下来需要探讨的问题，就是如何路由到具体的数据库和具体的表。以分片键路由到具体某一个数据库为例，常见的路由方式如下：\n\n#### **映射路由**\n\n映射路由，即新增一个库，新建一个路由映射表，存储分片键值和对应的库之间的映射关系。比如，键值为 1001，映射到 db01 这个数据库，如下图所示：\n\n![](https://oscimg.oschina.net/oscnet/up-cfd0ac43199f735ff69c5aa49842adaf1e4.png)\n\n映射方式，优点是映射方式可任意调整，扩容简单，但是存在一个比较严重的不足，就是映射库中的映射表的数据量异常巨大。我们本来的目标是要实现分库分表的功能，可是现在，映射库映射表相当于回到了分库分表之前的状态。所以，我们在实践中，没有采取这种方式。\n\n#### **分组路由**\n\n分组路由，即对分片的键值，进行分组，每组对应到一个具体的数据库。比如，键值为 1000 到 2000，则存储到 db01 这个数据库，如下图所示：\n\n![](https://oscimg.oschina.net/oscnet/up-e9961db378f4c5995994e452fe60a0ed897.png)\n\n分组方式，优点是扩容简单，实现简单，但是也存在一个比较严重的不足，是数据分布热点问题，比如在某一个时间内，分片键值为 2001，则在将来一段时间内，所有的数据流量，全部打到某一个库（db02）。这个问题，在互联网环境下，也比较严重，比如在一些促销活动中，订单量会有一个明显的飙升，这时候各个数据库不能达到分摊流量的效果，只有一个库在接收流量，会回到分库分表之前的状态。所以，我们也没有采取这种方式。\n\n#### **哈希路由**\n\n哈希路由，即对分片的键值，进行哈希，然后根据哈希结果，对应到一个具体的数据库。比如，键值为 1000，对其取哈希的结果为 01，则存储到 db01 这个数据库，如下图所示：\n\n![](https://oscimg.oschina.net/oscnet/up-d7875cca3405c6e251421a2a18f6f7be169.png)\n\n哈希方式，优点是分布均匀，无热点问题，但是反过来，数据扩容比较麻烦。因为在扩容过程中，需要调整哈希函数，随之带出一个数据迁移问题。互联网环境下，迁移过程中往往不能进行停服，所以就需要类似多库双写等方式进行过渡，比较麻烦。所以，在实践中也没有采取这种方式。\n\n#### **分组哈希路由**\n\n分组哈希路由，即对分片的键值，先进行分组，后再进行哈希。如下图所示：\n\n![](https://oscimg.oschina.net/oscnet/up-038c74382d3b91ed7a05b6c49c5435508b8.png)\n\n在实践中，我们结合了前面的几种方式，借鉴了他们的优点不足，而采用了此种方式。因为分组方式，能很方便的进行扩容，解决了数据扩容问题；哈希方式，能解决分布相对均匀，无单点数据库热点问题。\n\n### **3.4 技术中间件**\n\n分库分表的中间件选取，在行业内的方案还是比较多的，公司也有自己的实现。根据实现方式的不同，可以分为代理和非代理方式，下面列举了一些业界常见的中间件，如下表（截至于 2021-04-08）：\n\n![](https://oscimg.oschina.net/oscnet/up-0d958dbd08be1c973a60c3c077cf3e29e7c.JPEG)\n\n我们为什么最终选择了 ShardingSphere 呢？主要从这几个因素考虑：\n\n#### **技术环境**\n\n-   我们团队是 Java 体系下的，对 Java 中间件有一些偏爱\n    \n-   更偏向于轻量级组件，可以深入研究的组件\n    \n-   可能会需要一些个性定制化\n    \n\n#### **专业程度**\n\n-   取决于中间件由哪个团队进行维护，是否是名师打造，是否是行业标杆\n    \n-   更新迭代频率，最好是更新相对频繁，维护较积极的\n    \n-   流行度问题，偏向于流行度广、社区活跃的中间件\n    \n-   性能问题，性能能满足我们的要求\n    \n\n#### **使用成本**\n\n-   学习成本、入门成本和定制改造成本\n    \n-   弱浸入性，对业务能较少浸入\n    \n-   现有技术栈下的迁移成本，我们当前技术栈是 SSM 体系下\n    \n\n#### **运维成本**\n\n-   高可用、高稳定性\n    \n-   减少硬件资源，不希望再单独引入一个代理中间件，还要考虑运维成本\n    \n-   丰富的埋点、完善的监控\n    \n\n## **四、业务实践**\n\n在业务实践中，我们经历了从新库新表的设计，分库分表自建代理、服务收口、上游订单应用迁移，历史数据迁移等过程。\n\n### **4.1 新表模型**\n\n为了建立分库分表下的关联关系，和更加合理有效的结构，我们新申请了订单分库分表的几个库，设计了一套全新的表结构。表名以年份结尾、规范化表字段、适当增删了部分字段、不使用自增主键关联，采用业务唯一键进行关联等。\n\n表结构示例如下图：\n\n![](https://oscimg.oschina.net/oscnet/up-12ac37720f2e42ee55255c35d626231b6e2.png)\n\n### **4.2 服务收口**\n\n自建了一个分库分表数据库的服务代理 Dal-Sharding。每一个需要操作订单库的服务，都要通过代理服务进行操作数据库，达到服务的一个收口效果。同时，屏蔽了分库分表的复杂性，规范数据库的基本增删改查方法。\n\n![](https://oscimg.oschina.net/oscnet/up-c52f69ac309c952856c373a629d4143440f.png)\n\n### **4.3 平滑过渡**\n\n应用迁移过程中，为了保证应用的平滑过渡，我们新增了一些同步逻辑，来保证应用的顺利迁移，在应用迁移前后，对应用没有任何影响。未迁移的应用，可以读取到迁移后应用写入的订单数据；迁移后的应用，能读取到未迁移应用写入的订单数据。同时，统一实现了此逻辑，减少各个应用的迁移成本。\n\n**新老库双读**\n\n顾名思义，就是在读取的时候，两个库可能都要进行读取，即优先读取新库，如果能读到记录，直接返回；否则，再次读取老库记录，并返回结果。\n\n双读的基本过程如下：\n\n![](https://oscimg.oschina.net/oscnet/up-f1690cb1265e12a4523ea7aaf8f55c21620.png)\n\n新老库双读，保证了应用迁移过程中读取的低成本，上游应用不需要关心数据来源于新的库还是老的库，只要关心数据的读取即可，减少了切换新库和分库分表的逻辑，极大的减少了迁移的工作量。\n\n实践过程中，我们通过切面实现双读逻辑，将双读逻辑放入到切面中进行，减小新库的读取逻辑的侵入，方便后面实现对双读逻辑的移除调整。\n\n同时，新增一些配置，比如可以控制到哪些表需要进行双读，哪些表不需要双读等。\n\n**新老库双写**\n\n新老库双写，就是在写入新库成功后，异步写入到老库中。双写使得新老库都同时存在这些订单数据，尚未迁移通过代理服务操作数据库的应用得以正常的运作。\n\n双写的基本过程如下：\n\n![](https://oscimg.oschina.net/oscnet/up-2e4a7924da6ac7e366b115ad3b0eaceb9b5.png)\n\n双写其实有较多的方案，比如基于数据库的日志，通过监听解析数据库日志实现同步；也可以通过切面，实现双写；还可以通过定时任务进行同步；另外，结合到我们自己的订单业务，我们还可以通过订单事件（比如创单成功、出票成功、退票成功等），进行双写，同步数据到老库中。\n\n目前，我们经过考虑，没有通过数据库日志来实现，因为这样相当于把逻辑下沉到了数据库层面，从实现上不够灵活，同时，可能还会涉及到一些权限、排期等问题。实践中，我们采取其他三种方式，互补形式，进行双写。异步切面双写，保证了最大的时效性；订单事件，保证了核心节点的一致性；定时任务，保证了最终的一致性。\n\n跟双读一样，我们也支持配置控制到哪些表需要进行双写，那些表不需要双写等。\n\n**过渡迁移**\n\n有了前面的双读双写作为基础，迁移相对容易实行，我们采取逐个迁移的方式，比如，按照服务、按照渠道和按照供应进行迁移，将迁移工作进行拆解，减少影响面，追求稳健。一般分为三步走方式：\n\n1）第一阶段，先在新对接的供应商中进行迁移新库，因为新上线的供应商，订单量最少，同时哪怕出现了问题，不至于影响到之前的业务。\n\n2）再次迁移量比较少的线上业务，此类订单，有一些量，但是追求稳定，不能因为切换新库而产生影响。所以，将此类业务放到了第二阶段中进行。\n\n3）最后一步是，将量较大的业务，逐渐迁移到新库中，此类业务，需要在在有前面的保证后，方能进行迁移，保证订单的正常进行。\n\n![](https://oscimg.oschina.net/oscnet/up-60ddd27b3aaf8d0451b641ca02afcfad65b.png)\n\n### **4.4 数据迁移**\n\n数据迁移，即将数据，从老库迁移到新库，是新老库切换的一个必经过程。迁移的常规思路，一般是每个表一个个进行迁移，结合业务，我们没有采取此做法，而是从订单维度进行迁移。\n\n举个例子：假如订单库有 Order 表、OrderStation 表、OrderFare 表三个表，我们没有采取一个一个表分别进行迁移，而是根据订单号，以每一个订单的信息，进行同步。\n\n大致过程如下：\n\n1）开启一个定时任务，查询订单列表，取得订单号等基本订单信息。\n\n2）根据这个订单号，去分别查询订单的其他信息，取得一个完整的订单信息。\n\n3）校验订单是否已经完成同步，之前完成同步了则直接跳过，否则继续执行下一个订单号。\n\n4）将老库的完整的订单信息，映射成新库的对应的模型。\n\n5）将新的订单信息，同步写入到新库各个表中。\n\n6）继续执行下一个订单号，直到所有的订单号都完全同步结束。\n\n![](https://oscimg.oschina.net/oscnet/up-c900164b1e0238e7f40b0acfd8cac4fbf8b.png)\n\n### **4.5 完成效果**\n\n订单库经过一个全新的重构，目前已经在线上稳定运行，效果显著，达到了我们想要的效果。\n\n-   服务收口，将分库分表逻辑，收口到了一个服务中；\n    \n-   接口统一管理，统一对敏感字段进行加密；\n    \n-   功能灵活，提供丰富的功能，支持定制化；\n    \n-   分库分表路由透明，且基于主流技术，易于上手；\n    \n-   完善的监控，支持到表维度的监控；\n    \n\n## **五、****常见问题总结**\n\n### **5.1 分库分表典型问题**\n\n**问题1**：如何进行跨库操作，关联查询，跨库事务？\n\n**回答：**对于跨库操作，在订单主流程应用中，我们目前是禁止了比如跨库查询、跨库事务等操作的。对于跨库事务，因为根据订单号、创建年份路由，都是会路由到同一个数据库中，也不会存在跨库事务。同样对于跨库关联查询，也不会存在，往往都是根据订单来进行查询。同时，也可以适当进行冗余，比如存储车站编码的同时，多存储一个车站名称字段。\n\n**问题2**：如何进行分页查询？\n\n**回答**：目前在订单主流程应用中的分页查询，我们直接采用了 Sharding-JDBC 提供的最原始的分页方式，直接按照正常的分页 SQL，来进行查询分页即可。理由：主流程订单服务，比如出票系统，往往都是查询前面几页的订单，直接查询即可，不会存在很深的翻页。当然，对于要求较高的分页查询，可以去实现二次查询，来实现更加高效的分页查询。\n\n**问题3**：如何支持很复杂的统计查询？\n\n**回答**：专门增加了一个宽表，来满足那些很复杂查询的需求，将常用的查询信息，全部落到此表中，进而可以快速得到这些复杂查询的结果。\n\n### **5.2 API 方法问题**\n\n**问题：**服务收口后，如何满足业务各种不同的查询条件？\n\n**回答：**我们的 API 方法，相对固定，一般查询类只有两个方法，根据订单号查询，和根据 Condition 查询条件进行查询。对于各种不同的查询条件，则通过新增 Condition 的字段属性来实现，而不会新增各种查询方法。\n\n### **5.3 均匀问题**\n\n**问题：**在不同 group 中，数据会存在分布不均匀，存在热点问题？\n\n**回答：**是的，比如运行 5 年后，我们拓展成了 3 个 group，每一个 group 中存在 3 个库，那么此时，读写最多的应该是第三个 group。不过这种分布不均匀问题和热点问题，是可接受的，相当于前面的两个 group，可以作为历史归档 group，目前主要使用的 group 为第三个 group。\n\n随着业务的发展，你可以进行调配，比如业务发展迅速，那么相对合理的分配，往往不会是每个 group 是 3 个库，更可能是应该是，越往后 group 内的库越多。同时，因为每个 group 内是存在多个库，与之前的某一个库的热点问题是存在本质差别，而不用担心单数据库瓶颈问题，可以通过加库来实现扩展。\n\n### **5.4 Group 内路由问题**\n\n**问题：**对于仅根据订单号查询，在 group 内的路由过程是读取 group 内所有的表吗？\n\n**回答：**根据目前的设计，是的。目前是按年份分组，订单号不会存储其他信息，采用携程统一方式生成，也就是如果根据订单号查询，我们并不知道是存在于哪个表，则需要查询 group 内所有的表。对于此类问题，通常推荐做法是，可以适当增加因子，在订单号中，存储创建年份信息，这样就可以知道对应那个表了；也可以年份适当进行延伸，比如每 5 年一次分表，那么这样调整后，一个 group 内的表应该相对很少，可以极大加快查询效能。\n\n### **5.5 异步双写问题**\n\n**问题**：为什么双写过程，采用了多种方式结合的方式？\n\n**回答**：首先，切面方式，能最大限度满足订单同步的时效性。但是，在实践过程中，我们发现，异步切面双写，会存在多线程并发问题。因为在老库中，表的关联关系依赖于数据库的自增 ID，依赖于表的插入顺序，会存在关联失败的情况。所以，单纯依靠切面同步还不够，还需要更加稳健的方式，即定时任务（订单事件是不可靠消息事件，即可能会存在丢失情况）的方式，来保证数据库的一致性。\n\n**关于作者**\n\n瑞华：携程火车票研发团队，负责火车票业务的开发以及创新。火车票研发在多种交通线路联程联运算法、多种交通工具一站式预定、高并发方向不断地深入探索和创新，持续优化用户体验，提高效率，致力于为全球人民买全球火车票。\n\n","date":"2022-11-01","author":"瑞华","excerpt":"订单库经过一个全新的重构，目前已经在线上稳定运行，效果显著，达到了我们想要的效果。","metaDescription":null,"createdAt":"2022-11-03T09:11:41.372Z","updatedAt":"2022-11-03T09:11:42.622Z","publishedAt":"2022-11-03T09:11:42.619Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":33,"attributes":{"feature":true,"title":"覆盖 70% 核心业务，ShardingSphere 如何成为喜马拉雅架构演进的催化剂","content":"### 背景\n\n喜马拉雅成立之初，各个业务管理各自的数据库、缓存，各个业务都要了解中间件的各种部署情况，导致业务间的合作，需要运维、开发等方面的人工介入，效率较低，扩展困难，安全风险也很高，资源利用率也不高。喜马拉雅在发展中，逐渐意识到需要在公司层面，提供统一的定制化的数据访问平台的重要性。为此，我们推出了自己的 PaaS 化平台，PaaS 化就是对资源的使用做了统一的入口，业务只需要申请一个资源 ID，就能使用数据库，达到对资源使用的全部系统化，其中对数据库的访问我们基于 Apache ShardingSphere 来实现，并基于 Apache ShardingSphere 强大功能做些优化和增强。\n\n### 整体架构\n\n我们 PaaS 平台建设中，负责和数据层通信的 dal 层中间件我们叫 Arena，其中对数据库的访问我们叫 Arena-Jdbc。\n\nArena-Jdbc 层的能力基本是基于 Apache ShardingSphere 的能力建设，我们只是基于喜马拉雅需要的特性做了增强和优化，整体架构如下：\n\n![](https://img-blog.csdnimg.cn/img_convert/96d0bfa8558f18cfd3ce1544cae6f58d.png)​\n\n### Pull Frame\n\nConsul Pull Frame 是我们对 Consul 的配置自动拉起封装为统一的 Pull 框架，我们除了数据库，还有缓存，每种还有不同的使用方式，我们对不同的使用方式只需要实现对应的实现类和初始化，更新，切好这些接口就行，框架会统一把解析好的数据给到，具体一种场景不需要关心和 Consul 的交互，为后面的资源 PaaS 化提供了简单的接入能力。\n\n### 故障容灾\n\n· 自动重连\n\n我们对故障容灾在设计时就考虑了平时通用的一些故障场景，比如数据库 server 挂了，我们做自动重链，不需要业务做重启操作。\n\n· 本地快照\n\n本地快照是为了防止 Consul 不可用时，业务不能启动，所以我们在拉到远程配置后，会本地存储一份，在拉配置时，如果远程失败，就用本地的配置，保证 Consul 挂了，不影响业务，每次拉到新的配置时，会更新本地的快照。\n\n· 灰度更新\n\n灰度更新是为了支持配置变更时找灰度的逻辑，对于数据库层面的变更，是非常危险的，如果一下就全量变更，有可能会触发线上事故，所以通过灰度变更的机制，业务可以先选择一个容器实例来变更，没有问题后，再全量变更，把风险降到最低。\n\n· 密码安全\n\n没有 PaaS 化之前，我们的数据库密码都是 DBA 统一管理的，但 PaaS 化后，访问数据库的密码就存在配置文件中，如果明文，就太不安全，所以我们对密码统一做了加密处理，在 Arena-Jdbc 层统一做解密，确保密码不会泄露出去。\n\n### 统一数据源\n\n为了让业务做最低成本的改造，Arena-Jdbc 需要提供一个统一的数据源，不论上层用什么框架，不影响业务只需要替换数据源接入即可，对于数据库连接池我们默认使用 HikariCP DataSource 也支持个性化的业务，业务可以通过配置指定连接池。\n\n我们基于 Apache ShardingSphere 的连接池封装了一个我们自己的 DataSource，我们叫 ArenaDataSource，通过 ArenaDataSource 封装了各种不同场景聚合的使用一个 ArenaDataSource 支持三种使用方式：\n\n1.  支持原生直接连接\n2.  支持 Proxy 模式，也是 Apache ShardingSphere 的 Proxy\n3.  支持直接连接分库分表\n\n业务只需要一个 DataSource，即支持分库分表，也支持简单的直接连接的模式，这样的好处是业务以后要分库分表，就不再需要升级中间件了，为了彻底解决业务升级的成本，我们做了配置自动升级，就是你之前是简单直接链接使用，为了 PaaS 化，后来业务发展了，需要分库分表了，以及从分库分表需要多活部署了，这些都不要再升级依赖了，只需要配置动即可。\n\n### 资源动态变更\n\n资源动态变更是 PaaS 平台基本的能力，接入 PaaS 后，业务修改数据库的任何属性，都不再需要业务方代码变更，重新发布。\n\nApache ShardingSphere 也支持数据库属性的动态变更，我们基于自己的内部系统的特征，实现了基于 Consul 的资源变更通知，我们的资源存在 Consul。\n\nArena-Jdbc 支持对使用的资源做无损的变更，Arena-Jdbc 收到资源变更时，会先对新下发的资源做预热处理，预热后，再切换使用的数据源，切换成功后，再销毁老的数据源，业务无感知。\n\n如果新的资源预热失败，则不会做变更处理，保证下发的资源是可用的，规避错误下发的问题。\n\n扩容和缩容也是同理，一期数据需要运维手动迁移，迁移好了后，直接在 PaaS 平台下发新的配置即可，二期支持自动迁移数据和配置变更结合。\n\n同时支持 Proxy 的无损上下线机制，通过 PaaS 平台对 Proxy 的变更，把需要下线的 Proxy 节点去掉，通知 Arena-Jdbc，Arena-Jdbc 会把缩容的 Proxy 节点去掉，做到无损下线。\n\n### 读写分离\n\n读写分离我们完全基于 Apache ShardingSphere 的来实现，我们根据喜马拉雅业务的特性，对强制路由做了增强，不需要规则配置为 Hint 模式，只要线程上下文带有强制路由的标志，就可以路由到指定的库和表，不受分表规则的影响，我们重写了 ShardingStandardRoutingEngine 的 Sharding 时路由库和表的逻辑：\n\nprivate Collection<String> routeDataSources(final TableRule tableRule, final ShardingStrategy databaseShardingStrategy, final List<ShardingConditionValue> databaseShardingValues) {\n        //先判断是否存在Hint上下文路由标，如果有,则优先根据用户指定的规则路由库\n        Collection<Comparable<?>> databaseShardings = HintManager.getDatabaseShardingValues(tableRule.getLogicTable());\n        if (databaseShardings != null && databaseShardings.size() > 0) {\n            List<String> list = new ArrayList<>(4);\n            for (Comparable<?> databaseSharding : databaseShardings) {\n                list.add((String) databaseSharding);\n            }\n            if (log.isDebugEnabled()) {\n                log.debug(\"route dataSources, find HintManager, so hint to: {}\", list);\n            }\n            return list;\n        }\n        //没有Hint路由规则，则按Sharding 规则路由\n        if (databaseShardingValues.isEmpty()) {\n            return tableRule.getActualDatasourceNames();\n        }\n        Collection<String> result = new LinkedHashSet<>(databaseShardingStrategy.doSharding(tableRule.getActualDatasourceNames(), databaseShardingValues, properties));\n        Preconditions.checkState(!result.isEmpty(), \"no database route info\");\n        Preconditions.checkState(tableRule.getActualDatasourceNames().containsAll(result), \n                \"Some routed data sources do not belong to configured data sources. routed data sources: `%s`, configured data sources: `%s`\", result, tableRule.getActualDatasourceNames());\n        return result;\n    }\n\n路由表也是同样的逻辑，我们重写了 ShardingStandardRoutingEngine 的 routeTables 方法，和上面一样，先从 Hint 的上下文获取。这样通过上下文的方式能很好地满足业务个性化的路由规则，能和 Sharding 规则共存。\n\n### Database Plus\n\nApache ShardingSphere 除了提供基本的分库分表，读写分离的能力外，在上层还提供了很多的插件和扩展的机制，这让我们在基于数据库提供更偏向业务的能力非常容易，成本非常低，这叫 Database Plus。\n\nDatabase Plus 简单的说就是用 Apache ShardingSphere 的数据库中间件，不仅仅是提供了分库分表这一基本能力，通过对底层数据的封装为统一的交互标准插件模式，可以在上面实现很多业务的通用的场景的需求，比如喜马拉雅除了用到 Apache ShardingSphere 基础的能力外，我们也享受了 Database Plus 的威力，我们在它的基础上轻松实现了支持压测的影子库和影子表，数据加解密，机房级别容灾的同城双读，分布式唯一 ID。\n\n### 影子库和影子表\n\n影子库影子表我们对 Apache ShardingSphere 做了改动，Apache ShardingSphere 需要修改 SQL，我们认为对业务有改造成本，同时结合我们自己的压测平台，我们和业界一样，我们也实现了影子标记，通过全链路压测标的传递来判断是否路由到影子库/影子表，业务无需任何改造，即可使用影子库影子表来做压测，同时不需要在运行时对 SQL 改写，提升了性能，我们重写了 ShadowSQLRouter。\n\npublic class ArenaShadowSQLRouter extends ShadowSQLRouter {\n    @Override\n    public boolean isShadow(final SQLStatementContext<?> sqlStatementContext, final List<Object> parameters, final ShadowRule rule) {\n        if (sqlStatementContext instanceof InsertStatementContext || sqlStatementContext instanceof WhereAvailable\n            || sqlStatementContext instanceof UpdateStatementContext) {\n                 //这里就是判断是否有压测标，如果有，ShardingSphere则会找影子的逻辑。\n                return ArenaUtilities.checkPeakRequest();\n        }\n        return false;\n    }\n}\n\n通过 spi 的方式把我们自定义的 ArenaShadowSQLRouter 给 Apache ShardingSphere 加载使用，不得不说 Apache ShardingSphere 的插件设计很赞，很方便自定义和扩展。\n\n配置还是和 Apache ShardingSphere 的一样：\n\n配置影子库规则\n\\- !SHADOW\n    # true-影子表，false-影子库(默认)\n    enableShadowTable: true\n\n    # 源库名称(对应DataSources数据源配置中的名称)，影子库才需要配，影子表不需要配置\n    sourceDataSourceNames:  \n      \\- ds0           # 源库，与影子库shadow_ds0对应\n      \\- ds1           # 源库，与影子库shadow_ds1对应\n\n    # 影子库名称(对应dataSources数据源配置中的名称)，影子库才需要配，影子表不需要配置\n    shadowDataSourceNames:  \n      \\- shadow_ds0    # 影子库，与源库ds0对应\n      \\- shadow_ds1    # 影子库，与源库ds1对应\n\nenableShadowTable 我们新增了该属性，来确定是使用影子库还是影子表。\n\n### 影子库\n\n影子库，一定要填 sourceDataSourceNames 和 shadowDataSourceNames，enableShadowTable 不用设置，或者设置为 false。\n\nsourceDataSourceNames\n\n按顺序映射，一一对应：\n\nds --> shadow_ds\n\nds1--> shadow_ds1\n\n影子库/影子表是最后一个路由规则，如果发现有影子库/影子表，则根据实际的库找到对应的影子库/影子表，执行 SQL。\n\n### 同城多活\n\n基于喜马拉雅的业务特性，读多写少，我们只实现了对读业务的双机房部署，写业务还是路由到主机房。\n\n为了增强容灾能力，喜马拉雅搭建了双机房，同时承载业务流量，当一个机房故障时，可以把流量快速切换到另一个机房，我们在 dal 层设计上支持双写，这里充分利用了 Apache ShardingSphere 的读写分离功能，读和写可以配置独立的数据源，我们只需要在上面做了一层封装，在切换时候动态变更对应的数据源即可，为了切换时不影响业务的流量，我们是先预热新的数据源，再销毁老的数据源。\n\n架构图如下：\n\n![](https://img-blog.csdnimg.cn/img_convert/19f6cb8d352c0b53c2cd7b5ba0ec9003.png)​\n\n另外我们也对双写做了研究和探索，关键在于数据库的双向同步，基于阿里开源的 otter 做了改造，支持基于 gtid 模式同步，不依赖打标，打标会有性能开销，在一些业务做了试用。\n\n### 分布式唯一 ID\n\n分库分表后，唯一 id 是必须要满足的需求，Apache ShardingSphere 默认提供了 snowfake 和 uuid 算法，但不是很 db 时候的场景，db 需要保证顺序和格式，所以我们基于 Apache ShardingSphere 提供的接口，也实现自己的唯一 id 生成策略：数据分片后，不同 MySQL 实例生成全局唯一主键是非常棘手的问题。Arena-Jdbc 实现了 Apache ShardingSphere 的分布式主键生成器接口，通过集成喜马拉雅内部的全局唯一 id 生成服务，提供了适用于喜马拉雅内部的自增主键生成算法- BoushId 主键生成策略。\n\n### 监控和报警\n\n做一个数据库中间件，监控是必不可少的部分，就像我们的眼睛，没有监控就是瞎抓，以及对异常情况的报警也是非常重要的部分，只有完善的监控和报警才能算是一个完整的产品，得益于 Apache ShardingSphere 在设计时就提供了钩子，我们能非常小的成本就能实现对 SQL 层面的监控和报警。\n\nArena-Jdbc 客户端通过钩子回调，从多维度数据来分析使用数据库的运行情况，以 30s 为一次统计周期，每个周期统计的数据包括：MySQL 总请求量，新增、删除、修改和查询的请求量，失败的请求量和慢请求量，影子库的流量，以及统计响应时间的 TP 百分比，还有连接池的等待时间、建连时间、连接数等信息。这些指标会发送给专门的收集指标服务，并持久化到时序数据库，PaaS 平台可以从时序数据库中查询数据，展示给各个业务，对于异常 SQL 和慢 SQL，做报警等后续处理。\n\n![](https://img-blog.csdnimg.cn/img_convert/104ee654eee8dc89263510fa9f31aad6.png)​\n\n### 其他\n\n我们除了基于 Apache ShardingSphere 实现上述关键特性外，我们还对 Apache ShardingSphere 做了一些优化和改进，以更适合喜马拉雅的业务。\n\n· 优化分片规则，启动时，如果分片的真实表不存在的情况则报错，将配置错误前置；\n\n· 有的业务方有几百，甚至几千的分表，这种情况下，由于 Apache ShardingSphere 中的联邦查询需要依次扫表，启动速度很慢，达到了分钟级别。针对这种情况，我们新增了 props 配置项，不再初始化联邦查询，大大加快了启动速度，并且在使用中，也没有用联邦查询；\n\n· 优化了 Apache ShardingSphere，执行 SQL 异常不报错误的情况；\n\n· 由于有的业务方，对重要的表采用了大写的表名和列名，我们去掉 Apache ShardingSphere 中，对配置中的大写的表名列名强制小写的情况，允许大写的表名和列名；\n\n· 新增了 props 配置项，可以调节 Apache ShardingSphere 的编译缓存的大小；\n\n· 优化 Apache ShardingSphere 复合分片算法，精确匹配分片字段；\n\n· 在 ComplexShardingStrategyConfiguration 中，添加 shardingColumnList 字段，修复 Apache ShardingSphere 批量 insert 不返回主键的问题，这个问题在 mybatis-plus 中比较常见；\n\n· 不分片的表，支持使用默认的主键 id 生成策略。\n\n### 总结\n\n基于 Apache ShardingSphere 实现的数据库中间件 Arena-Jdbc，经过半年的时间，已经覆盖了喜马拉雅的 70% 的核心业务，目前没有发现任何问题，表现的非常稳定，通过和我们的 PaaS 平台结合，业务也非常愿意接入，另外我们使用 Apache ShardingSphere 时，社区还没有发布 stable 的版本，所以我们在使用过程中也遇到了些问题，基本上我们都解决了，有的社区也有对应的解决方案，得益于社区非常活跃，我们以后也希望把我们做的一些 feature 能回馈到社区，为 Apache ShardingSphere 的发展做出一点点小贡献。\n\n非常感谢基础架构团队胡建华、彭荣新提出的宝贵意见和建议，感谢喜马拉雅基础小伙伴们在项目推广过程中的大力支持，让 Apache ShardingSphere 在喜马拉雅生根发芽。\n\n非常感谢亮哥亲自来喜马拉雅对 Apache ShardingSphere 的技术内幕和规则做了一次全面的分享，非常关心我们在使用 Apache ShardingSphere 过程中遇到的问题，在现场对小伙伴提的问题都一一作了深入的解答，非常感谢亮哥，祝 Apache ShardingSphere 越来越好。","date":"2022-11-01","author":"沈辉 ","excerpt":"基于 Apache ShardingSphere 强大功能做优化和增强","metaDescription":null,"createdAt":"2022-11-03T09:45:09.484Z","updatedAt":"2022-11-03T09:45:10.507Z","publishedAt":"2022-11-03T09:45:10.504Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":37,"attributes":{"feature":true,"title":"ShardingSphere 在数十个联通政务热线场景中的应用：稳定、高效、可复制","content":"# **一、项目背景**\n\n  随着“互联网+政务服务”政策的持续发力，越来越多的地方政府开始着手打造“互联网+”环境下的政务服务平台。\n\n  近年来，联通数科先后助力数十个政务热线平台上线，以提升政务服务能力，部分项目如下图。\n![up-20ec9e57d65c64dcf55a4277105b5ff0488.webp](https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/up_20ec9e57d65c64dcf55a4277105b5ff0488_6143b4132c.webp)\n\n  政务热线业务有并发高、数据量大的特点，我们每拨打一次政务热线，即产生一条工单记录。相比以往，疫情期间的热线业务量更是翻了数倍。\n\n  **在政务热线的工单模块中，为满足大规模、海量数据以及高稳定的业务需求，联通数科采用 ShardingSphere 对工单信息进行分片存储，在政民互动的桥梁中构建坚固的桥墩。**\n\n# **二、业务挑战**\n\n  政务热线是政府与企业、群众互动的主要渠道，以单一号码面对公众，提供 7*24 服务。除受理工单外，还包括了话务、知识库、语音对答及智能坐席等业务。\n\n  在系统规划阶段，数据库层作为业务系统的基石，技术选型过程更为谨慎。系统的稳定是硬性指标，其次是性能，性能直接影响热线业务的服务能力，同时还要具备易于维护和管理的特点，便于系统升级及后台维护。\n\n  在数据库架构选型中，政务热线系统对架构要求如下：\n\n  - **成熟稳定**\n  - **高性能**\n  - **易维护**\n  - **业务代码低耦合**\n\n# **三、选择 ShardingSphere 的理由**\n\n  **经过技术团队多轮调研测试，包括稳定性、产品功能、接入方式及性能等，最终选择“ShardingSphere + MySQL”分布式数据库解决方案。**\n\n  - **理念先进**\n\n  随业务应用场景多元化发展，数据库已呈碎片化趋势，**「Database Plus」**旨在数据库上层建立标准和生态，为数据库提供增量能力，可防止数据库绑定、实现自主可控，突破原有数据库功能局限，提升数据基础设施整体功能上限。\n\n  - **成熟稳定/案例多**\n\n  ShardingSphere 于 2016 年开源，经过多年的研发迭代，产品历经多个头部互联网大型场景打磨，成熟稳定、值得信赖。\n\n  - **功能全面**\n\n  ShardingSphere 除了具备 Sharding 能力外，数据加解密和影子库的功能也是作为联通技术团队评估的指标，通过一套技术栈既可解决安全和压测的场景需求。ShardingSphere 为政务热线系统建设及架构扩展，提供了更多的能力支持。\n\n  - **高兼容/接入方便**\n\n  ShardingSphere 与 MySQL 协议兼容，语法支持度也非常高，在开发过程中几乎不用考虑 SQL 的支持问题，接入非常方便。\n\n  - **高性能**\n\n  政务热线忙时有数千坐席在线，工单记录以千万记录为起始单位，对性能有明确的要求。ShardingSphere-JDBC 定位为轻量级的 Java 框架，压测结果数据理想，可满足政务系统服务能力要求。\n\n  - **易运维**\n\n  在 JDBC 良好的性能基础之上，ShardingSphere 还提供了对运维人员友好的 Proxy，使用常用的客户端即可直接访问，上手维护较容易。\n\n  除了以上五个维度的权衡，选型中也参考了其他数据库中间件产品，以下表格是 ShardingSphere 和 Mycat 的比对。\n\n  |              | **Apache ShardingSphere**                   | **Mycat**                                     |\n  | ------------ | ------------------------------------------- | --------------------------------------------- |\n  | 开源协议     | Apache-2.0                                  | GPL-2.0/GPL-3.0                               |\n  | 接入端       | 客户端/代理端                               | 代理端                                        |\n  | 数据库支持   | 客户端：任意代理端：MySQL/PG/openGauss      | MySQL、Oracle、DB2、SQL Server、PostgreSQL 等 |\n  | 性能         | 客户端：损耗低代理端：损耗高                | 损耗高                                        |\n  | 读写分离     | 有                                          | 有                                            |\n  | 数据加密     | 有                                          | 无                                            |\n  | 全链路压测   | 有                                          | 无                                            |\n  | 社区成熟度   | Apache 软件基金会信通院可信认证社区活跃度高 | 个人/三方社区社区活跃度一般                   |\n  | 商业公司支持 | 有                                          | 无                                            |\n\n# **四、解决方案**\n\n  由于“ShardingSphere+MySQL ”具备稳定、易用及性能极致等特点，该方案在联通数科的多个政务热线项目中复制使用。\n\n  - **部署方式**\n\n  ShardingSphere-JDBC 和 ShardingSphere-Proxy 混合使用的方式，兼顾了系统性能和可维护性。\n\n  - **分片键/分片算法**\n\n  政务热线的工单表相对较大，且使用逻辑简单，因此在分片设计上，选取工单 ID 作为分片键，通过哈希算法分布数据。\n\n  - **迁移**\n\n  新建项目不涉及历史数据，因此无需考虑数据迁移环节，上线即直接使用。\n\n  - **实施过程**\n\n  回忆数个已上线的项目，由于技术方案评估较全面，且多次复制方案，实施过程并没有遇到“坑”。\n\n  - **系统架构**\n\n  政务热线业务模块采用了微服务架构，节点在十几到几十不等，以具体省市规模决定。在数据库层，使用物理机部署 3 节点 MHA，每个政务热线系统一般需要 4 套 MHA 来支撑业务。\n\n![up-6eb6faa996c81e6de688a2dd9e204f730e6.webp](https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/up_6eb6faa996c81e6de688a2dd9e204f730e6_e867e826e4.webp)\n\n# **五、用户收益**\n\n  - **业务角度**\n    - 支持客户端模式，最大化保证了政务热线对性能的要求；\n    - 系统稳定，为业务连续性带来有力的支撑。\n  - **研发角度**\n    - 兼容良好，兼容主流数据库 SQL 语法，对接即可使用；\n    - 功能全面，读写分离、数据加密及全链路压测方案，系统扩展性强。\n  - **运维角度**\n    - 支持代理端模式，为运维人员提供了良好的维护入口。\n  - **技术选型角度**\n    - 防止数据库绑定，为未来升级提供了足够的灵活度支撑。\n\n# **六、总结**\n\n  通过联通数科政务热线的多个案例，验证了 ShardingSphere 在政务场景的能力，也进一步说明了 ShardingSphere 无行业限制的特点。\n  目前我国数字政府建设已进入全面提速阶段，联通数科和 ShardingSphere 通力合作，希望助力更多政务热线项目落地，打造更多高效、便捷和安全的政务热线。","date":"2022-07-11","author":"SphereEx","excerpt":"联通数科整合中国联通“云、大、物、智、链、安”（云计算、大数据、物联网、人工智能、区块链、安全）等能力，面向智慧城市、数字政府、工业互联网、生态环保、文化旅游等领域，联通数科已积累大量行业标杆案例，成功为客户提供更加丰富、专业的产品和服务。","metaDescription":null,"createdAt":"2023-03-28T07:23:33.986Z","updatedAt":"2023-08-25T06:21:38.774Z","publishedAt":"2023-03-28T07:23:44.797Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":41,"attributes":{"feature":true,"title":"喜马拉雅基于Apache ShardingSphere实践","content":"# **背景** \n\n \n\n喜马拉雅成立之初，各个业务管理各自的数据库、缓存，个业务都要了解中间件的各种部署情况，导致业务间的合作，需要运维、开发等方面的人工介入，效率较低，扩展困难，安全风险也很高，资源利用率也不高。喜马拉雅在发展中，逐渐意识到需要在公司层面，提供统一的定制化的数据访问平台的重要性。为此，我们推出了自己的PaaS化平台，PaaS化就是对资源的使用做了统一的入口，业务只需要申请一个资源ID，就能使用数据库，达到对资源使用的全部系统化，其中对数据库的访问我们基于 Apache ShardingSphere 来实现，并基于 Apache ShardingSphere 强大功能做些优化和增强。\n\n \n\n# **整体架构**\n\n \n\n我们PaaS平台建设中，负责和数据层通信的dal层中间件我们叫Arena,其中对数据库的访问我们叫Arean-Jdbc\n\nArena-Jdbc 层的能力基本是基于 Apache ShardingSphere 的能力建设，我们只是基于喜马拉雅需要的特性做了增强和优化,整体架构如下：\n\n \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=NmU5ZDNhMjRlNDczZjdiYzg3NTE0Yzg4OTc5N2Q0M2ZfUzg3c3FCbkozTG0zUzZDR1dFb1dHT29Od3Nod0VWQmRfVG9rZW46Ym94Y25UY1VJQlcxeG1qZ2RVU01aUlIwQjdmXzE2ODAwNzcwNDk6MTY4MDA4MDY0OV9WNA)\n\n \n\n# **Pull Frame**\n\n \n\nConsul Pull Frame 是我们对Consul 的配置自动拉起封装为统一的Pull框架，我们除了数据库，还有缓存，每种还有不同的使用方式，我们对不同的使用方式只需要实现对应的实现类和初始化，更新，切好这些接口就行，框架会统一把解析好的数据给到，具体一种场景不需要关心和Consul的交互，为后面的资源PaaS化提供了简单的接入能力。\n\n \n\n# **故障容灾**\n\n \n\n***· 自动重连***\n\n我们对故障容灾在设计时就考虑了平时通用的一些故障场景，比如数据库server 挂了，我们最做自动重链，不需要业务做重启操作。\n\n \n\n***· 本地快照***\n\n本地快照是为了防止Consul不可用时，业务不能启动，所以我们在拉到远程配置后，会本地存储一份，在拉配置时，如果远程失败，就用本地的配置，保证Consul挂了，不影响业务，每次拉到新的配置时，会更新本地的快照。\n\n \n\n***· 灰度更新***\n\n灰度更新是为了支持配置变更时找灰度的逻辑，对于数据库层面的变更，是非常危险的，如果一下就全量变更，有可能会触发线上事故，所以通过灰度变更的机制，业务可以先选择一个容器实例来变更，没有问题后，再全量变更，把风险降到最低。\n\n \n\n***· 密码安全***\n\n没有PaaS化之前，我们的数据库密码都是dba统一管理的，但PaaS化后，访问数据库的密码就存在配置文件中，如果明文，就太不安全，所以我们对密码统一做了加密处理，在 Arean-Jdbc 层统一做解密，确保密码不回泄露出去。\n\n \n\n# **统一数据源**\n\n \n\n为了让业务做最低成本的改造，Arena-jdbc 需要提供一个统一的数据源，不论上层用什么框架，不影响,业务只需要替换数据源接入即可，对于数据库连接池我们默认使用 HikariCP  DataSource 也支持个性化的业务，业务可以通过配置指定连接池。\n\n我们基于Apache ShardingSphere 的连接池封装了一个我们自己的DataSource,我们叫ArenaDataSource，通过ArenaDataSource封装了各种不同场景聚合，的使用一个 ArenaDataSource 支持三种使用方式：\n\n1. 支持原生直接连接\n\n2. 支持Proxy模式，也是Apache ShardingSphere的proxy\n\n3. 支持直接连接分库分表\n\n业务只需要一个 DataSource，即支持分库分表，也支持简单的直接连接的模式，这样的好处是业务以后要分库分表，就不再需要升级中间件了，为了彻底解决业务升级的成本，我们做了配置自动升级，就是你之前是简单直接链接使用，为了PaaS化，后来业务发展了，需要分库分表了，以及从分库分表需要多活部署了，这些都不要再升级依赖了，只需要配置动即可。\n\n\n\n# **资源动态变更**\n\n \n\n资源动态变更是PaaS平台基本的能力，接入PaaS后，业务修改数据库的任何属性，都不再需要业务方代码变更，重新发布 \n\nApache ShardingSphere 也支持数据库属性的动态变更，我们基于自己的内部系统的特征，实现了基于Consul的资源变根通知。我们的资源存在Consul。\n\nArena-Jdbc 支持对使用的资源做无损的变更，Arena-Jdbc 收到资源变更时，会先对新下发的资源做预热处理，预热后，再切换使用的数据源，切换成功后，再销毁老的数据源，业务无感知。\n\n如果新的资源预热失败，则不会做变更处理，保证下发的资源时可用的，规避错误下发的问题。\n\n扩容和缩容也是同理，一期数据需要运维手动迁移，迁移好了后，直接在PaaS平台下发新的配置即可，二期支持自动迁移数据和配置变更结合。\n\n同时支持Proxy的无损上下线机制，通过PaaS平台对Proxy的变更，把需要下线的Proxy节点去掉，通知 Arena-Jdbc，Arena-Jdbc 会把缩容的Proxy节点去掉，做到无损下线。\n\n \n\n# **读写分离**\n\n \n\n读写分离我们完全基于Apache ShardingSphere的来实现，我们根据喜马拉雅业务的特性，对强制路由做了增强，不需要规则配置为 Hint 模式，只要线程上下文带有强制路由的标志，就可以路由到指定的库和表，不受分表规则的影响，我们重写了ShardingStandardRoutingEngine的Sharding时路由库和表的逻辑：\n\n```java\nprivate Collection<String> routeDataSources(final TableRule tableRule, final ShardingStrategy databaseShardingStrategy, final List<ShardingConditionValue> databaseShardingValues) {\n        //先判断是否存在Hint上下文路由标，如果有,则优先根据用户指定的规则路由库\n        Collection<Comparable<?>> databaseShardings = HintManager.getDatabaseShardingValues(tableRule.getLogicTable());\n        if (databaseShardings != null && databaseShardings.size() > 0) {\n            List<String> list = new ArrayList<>(4);\n            for (Comparable<?> databaseSharding : databaseShardings) {\n                list.add((String) databaseSharding);\n            }\n            if (log.isDebugEnabled()) {\n                log.debug(\"route dataSources, find HintManager, so hint to: {}\", list);\n            }\n            return list;\n        }\n        //没有Hint路由规则，则按Sharding 规则路由\n        if (databaseShardingValues.isEmpty()) {\n            return tableRule.getActualDatasourceNames();\n        }\n        Collection<String> result = new LinkedHashSet<>(databaseShardingStrategy.doSharding(tableRule.getActualDatasourceNames(), databaseShardingValues, properties));\n        Preconditions.checkState(!result.isEmpty(), \"no database route info\");\n        Preconditions.checkState(tableRule.getActualDatasourceNames().containsAll(result), \n                \"Some routed data sources do not belong to configured data sources. routed data sources: `%s`, configured data sources: `%s`\", result, tableRule.getActualDatasourceNames());\n        return result;\n    }\n```\n\n \n\n路由表也是同样的逻辑，我们重写了ShardingStandardRoutingEngine的routeTables方法，和上面一样。先从Hint的上下文获取。这样通过上下文的方式能很好的满足业务个性化的路由规则，能和Sharding规则共存。\n\n \n\n# **Database Plus**\n\n \n\nApache ShardingSphere 除了提供基本的分库分表，读写分离的能力外，在上层还提供了很多的插件和扩展的机制，这让我们在基于数据库提供更偏向业务的起的能力非常容易，成本非常低,这叫 Database Plus\n\nDatabase Plus 简单的说就是你用 Apache ShardingSphere 的数据库中间件，不仅仅是提供了分库分表这一基本能力，通过对底层数据的封装为统一的交互标准插件模式，可以在上面实现很多业务的通用的场景的需求，比如喜马拉雅除了用到 Apache ShardingSphere 基础的能力外，我们也享受了 Database Plus 的威力，我们在它的基础上轻松实现了支持压测的影子库和影子表，数据加解密，机房级别容灾的同城双读，分布式唯一ID。\n\n \n\n# **影子库和影子表**\n\n \n\n影子库影子表我们对Apache ShardingSphere做了改动，Apache ShardingSphere需要修改sql，我们认为对业务有改造成本，同时结合我们自己的压测平台，我们和业界一样，我们也实现了影子标记，通过全链路压测标的传递来判断是否路由到影子库/影子表，业务无需任何改造，即可使用影子库影子表来做压测，同时不需要在运行时对sql改写，提升了性能，我们重写了ShadowSQLRouter,\n\n```java\npublic class ArenaShadowSQLRouter extends ShadowSQLRouter {\n    @Override\n    public boolean isShadow(final SQLStatementContext<?> sqlStatementContext, final List<Object> parameters, final ShadowRule rule) {\n        if (sqlStatementContext instanceof InsertStatementContext || sqlStatementContext instanceof WhereAvailable\n            || sqlStatementContext instanceof UpdateStatementContext) {\n                 //这里就是判断是否有压测标，如果有，ShardingSphere则会找影子的逻辑。\n                return ArenaUtilities.checkPeakRequest();\n        }\n        return false;\n    }\n}\n```\n\n \n\n通过spi的方式把我们自定义的ArenaShadowSQLRouter给Apache ShardingSphere加载使用，不得不说Apache ShardingSphere的插件设计很赞，很方便自定义和扩展。\n\n配置还是和Apache ShardingSphere的一样：\n\n```yaml\n# 配置影子库规则\n- !SHADOW\n    # true-影子表，false-影子库(默认)\n    enableShadowTable: true \n\n    # 源库名称(对应DataSources数据源配置中的名称)，影子库才需要配，影子表不需要配置\n    sourceDataSourceNames:  \n      - ds0           # 源库，与影子库shadow_ds0对应\n      - ds1           # 源库，与影子库shadow_ds1对应\n\n    # 影子库名称(对应dataSources数据源配置中的名称)，影子库才需要配，影子表不需要配置\n    shadowDataSourceNames:  \n      - shadow_ds0    # 影子库，与源库ds0对应\n      - shadow_ds1    # 影子库，与源库ds1对应\n```\n\n \n\nenableShadowTable 我们新增了该属性，来确定是使用影子库还是影子表。\n\n \n\n# **影子库**\n\n \n\n影子库，一定要填sourceDataSourceNames和shadowDataSourceNames，enableShadowTable不用设置，或者设置为false。\n\n \n\nsourceDataSourceNames\n\n按顺序映射，一一对应：\n\nds --> shadow_ds\n\nds1--> shadow_ds1\n\n影子库/影子表是最后一个路由规则，如果发现有影子库/影子表，则根据实际的库找到对应的影子库/影子表，执行sql\n\n \n\n# **同城多活**\n\n \n\n基于喜马拉雅的业务特性，读多写少，我们只实现了对读业务的双机房部署，写业务还是路由到主机房。\n\n为了增强容灾能力，喜马拉雅搭建了双机房，同时承载业务流量，当一个机房故障时，可以把流量快速切换到另一个机房，我们在dal层设计上支持双写，这里充分利用了 Apache ShardingSphere 的读写分离功能，读和写可以配置独立的数据源，我们只需要在上面做了一层封装，在切换时候动态变更对应的数据源即可，为了切换时不影响业务的流量，我们是先预热新的数据源，再销毁老的数据源。\n\n架构图如下：\n\n \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ODgzZmMyZDliZDcwMTViYTIzYmE5MGE1MWE4YWJjNjBfQVU2NXhzWkIxTzdwZTJqT3MxNVNuRmdENUNqUExhRHNfVG9rZW46Ym94Y250TkVLaXJ1dzJZSlNnTDdlMkJOWk9lXzE2ODAwNzcwNDk6MTY4MDA4MDY0OV9WNA)\n\n \n\n另外我们也对双写做了研究和探索，关键在于数据库的双向同步，基于阿里开源的 otter 做了改造，支持基于 gtid 模式同步，不依赖打标，打标回有性能开销，在一些业务做了试用。\n\n \n\n# **分布式唯一ID**\n\n \n\n分库分表后，唯一id是必须要满足的需求，Apache ShardingSphere默认提供了snowfake和uuid算法，但不是很时候db的场景，db需要保证顺序和格式，所以我们基于Apache ShardingSphere提供的接口，也实现自己的唯一id生成策略：数据分片后，不同Mysql实例生成全局唯一主键是非常棘手的问题。Arena-Jdbc 实现了Apache ShardingSphere的分布式主键生成器接口，通过集成喜马拉雅内部的全局唯一id生成服务，提供了适用于喜马拉雅内部的自增主键生成算法-BoushId主键生成策略。\n\n \n\n# **监控和报警**\n\n \n\n做一个数据库中间件，监控是必不可少的部分，就像我们的眼睛，没有监控就是瞎抓，以及对异常情况的报警也是非常重要的部分，只有完善的监控和报警才能算是一个完整的产品，得益于 Apache ShardingSphere 在设计时就提供了钩子，我们能非常小的成本就能实现对sql层面的监控和报警。\n\n \n\nArena-Jdbc 客户端通过钩子回调，从多维度数据来分析使用数据库的运行情况，以30s为一次统计周期，每个周期统计的数据包括：Mysql总请求量，新增、删除、修改和查询的请求量，失败的请求量和慢请求量，影子库的流量，以及统计响应时间的TP百分比，还有连接池的等待时间、建连时间、连接数等信息。这些指标会发送给专门的收集指标服务，并持久化到时序数据库，PaaS平台可以从时序数据库中查询数据，展示给各个业务，对于异常sql和慢sql，做报警等后续处理。\n\n \n\n# **其他**\n\n \n\n我们除了基于Apache ShardingSphere实现上述关键特性外，我们还对Apache ShardingSphere做了一些优化和改进，以更适合喜马拉雅的业务。\n\n \n\n· 优化分片规则，启动时，如果分片的真实表不存在的情况则报错，将配置错误前置\n\n \n\n· 有的业务方有几百，甚至几千的分表，这种情况下，由于Apache ShardingSphere中的联邦查询需要依次扫表，启动速度很慢，达到了分钟级别。针对这种情况，我们新增了props配置项，不再初始化联邦查询，打打加快了启动速度，并且再使用中，也没有用联邦查询\n\n \n\n· 优化了Apache ShardingSphere，执行sql异常不报错误的情况\n\n \n\n· 由于有的业务方，对重要的表采用了大写的表名和列名，我们去掉Apache ShardingSphere中，对配置中的大写的表名列名强制小写的情况，允许大写的表名和列名\n\n \n\n· 新增了props配置项，可以调节Apache ShardingSphere的编译缓存的大小\n\n \n\n· 优化Apache ShardingSphere复合分片算法，精确匹配分片字段\n\n \n\n在ComplexShardingStrategyConfiguration中，添加shardingColumnList字段：修复Apache ShardingSphere，批量insert，不返回主键的问题，这个问题在mybatis-plus中比较常见\n\n \n\n· 不分片的表，支持使用默认的主键id生成策略\n\n \n\n# **总结**\n\n \n\n基于Apache ShardingSphere实现的数据库中间件Arena-Jdbc，经过半年的时间，已经覆盖了喜马拉雅的70%的核心业务，目前没有发现任何问题，表现的非常稳定，通过和我们的PaaS平台结合，业务也非常愿意接入，另外我们使用Apache ShardingSphere时，社区还没有发布stable的版本，所以我们在使用过程中也遇到了些问题，基本上我们都解决了，有的社区也有对应的解决方案，得益于社区非常活跃，我们以后也希望把我们做的一些feature能回馈到社区，为Apache ShardingSphere的发展做出一点点小贡献。\n\n非常感谢基础架构团队胡建华、彭荣新提出的宝贵意见和建议，感谢喜马拉雅基础小伙伴们在项目推广过程中的大力支持，让Apache ShardingSphere在喜马拉雅生根发芽。\n\n非常感谢亮哥亲自来喜马拉雅对Apache ShardingSphere的技术内幕和规则做了一次全面的分享，非常关心我们在使用Apache ShardingSphere过程中遇到的问题，在现场对小伙伴提的问题都一一作了 深入的解答，非常感谢亮哥，祝Apache ShardingSphere越来越好。","date":"2022-05-11","author":"沈辉  基础架构","excerpt":"基于Apache ShardingSphere实现的数据库中间件Arena-Jdbc，经过半年的时间，已经覆盖了喜马拉雅的70%的核心业务，目前没有发现任何问题，表现的非常稳定，通过和我们的PaaS平台结合，业务也非常愿意接入","metaDescription":null,"createdAt":"2023-03-29T08:09:25.219Z","updatedAt":"2023-03-29T08:09:26.226Z","publishedAt":"2023-03-29T08:09:26.216Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":13,"attributes":{"feature":true,"title":"ShardingSphere 助力当当 WMS：订单效率提升 30%、节约成本上千万","content":">Apache ShardingSphere 简单易用且成熟稳定，当当自研 WMS 如虎添翼，在供应链系统改革中起到了至关重要的作用，是我们的杀手锏。\n——李勇，当当 WMS 技术负责人\n\n>Apache ShardingSphere 5.0.0 GA 已于 2021 年 11 月发布，5.1.0 版本也于上周发布。历经两年的迭代与打磨，可插拔生态趋近成熟，标志着 Apache ShardingSphere 从分库分表中间件蜕变为以 Database Plus 为理念的分布式数据库生态系统。\n\n近年来，随着“建设社会主义文化强国”政策的持续落实，全民阅读在移动互联快速发展中得到大力倡导，阅读给当代人提供了丰富的精神食粮。当当成立于 1999 年底，作为新技术与传统行业的结合体，当当融合了互联网新技术和传统图书行业，二十余年深耕图书领域助力全民阅读。\n\n当当在刚成立的几年，也是中国互联网腾飞的阶段，电商逐渐多元化发展的趋势迎来了行业野蛮生长期，各电商平台的发展如逆水行舟，不进则退。当当对战略和管理等方面进行调整的同时，也对技术架构层进行了硬核升级。\n\n当当成立之初的仓储物流系统并非自研，随着业务体量、技术实力等多方面的变化，为了满足业务快速发展需要，当当决定重构 WMS（Warehouse Management System）和 TMS（Transportation Management System）。硬件层使用 x86 取代小机，数据层由传统集中式转型为更灵活的分布式架构。如何解决海量仓储货品信息问题，是重构待解决的核心问题之一。互联网技术栈比较常用的分库分表被率先尝试使用，但由于当时开源数据库中间件的成熟度及功能方面不满足预期，当当决定自研分库分表产品，Sharding-JDBC 就这样应运而生，为“连接”带来更多可能。\n\n**自研 WMS 的上线完成了仓储层智能改革，五年、十余仓，Apache ShardingSphere 同 WMS 完美支撑了数个“读书节”、“618”和“11.11”等大型促销活动。**\n\n\n## 业务挑战\n\n当当在自研 WMS 之前使用的是第三方仓储管理系统，其数据库是基于 IBM 小机的 Oracle，结合当当业务体量以及电商仓储物流在促销期间密集请求的业务特点，传统集中式数据库架构给 WMS 带来了极大的挑战。\n\n在业务快速发展的冲击下，数据库计算和存储能力的上限越发明显，Scale-up 方案难以满足超级促销期间的业务需求，技术团队需要对业务层多次做拆分调整来缓解“存算”瓶颈，产能存在受限的风险。\n\n- 计算、存储能力双受限\n\n传统集中式架构扩展能力相对较差，数据库的计算和存储能力成为明显瓶颈；\n\n- 业务开发维护成本高昂\n\n在扩展能力相对差的基础上，业务侧需要不断的去让步做拆分，开发和维护成本持续攀升；\n\n- 系统架构开放度不足\n\n系统架构开放度不足将会使系统面临灵活度欠佳、功能缺失以及转型难度高等问题，基于现有架构难以快速响应新业务的开展及相关政策的调整，如云原生、SQL 审计、数据脱敏以及分布式治理等多方面需求。\n\n\n## 解决方案\n\n当当技术部结合现状及未来发展等多维度的调研，最终确定了自研 WMS 的方案。在硬件方面全面采用通用的 x86 设备替换 IBM 小型机，数据库层面引入开源数据库 MySQL 来替换 Oracle。当时已有多个开源数据库中间件，但成熟度和功能方面与预期不符，因此选择了自研数据库中间件，Sharding-JDBC 闪亮登场。\n\nShardingSphere-JDBC 定位为轻量级 Java 框架，在 Java 的 JDBC 层提供额外服务。具备轻量、高效、易用以及高兼容等特点。它使用客户端直连数据库，以 jar 包形式提供服务，无需额外部署和依赖，可理解为增强版的 JDBC 驱动，完全兼容 JDBC 和各种 ORM 框架。\n\n- 适用于任何基于 JDBC 的 ORM 框架，如：JPA、Hibernate、Mybatis、Spring JDBC Template 或直接使用 JDBC；     \n\n- 支持任何第三方的数据库连接池，如：DBCP、C3P0、BoneCP、HikariCP 等；\n\n- 支持任意实现 JDBC 规范的数据库，目前支持 MySQL、PostgreSQL、Oracle、SQL Server 以及任何可使用 JDBC 访问的数据库。\n\n![当当1.png](https://official-media.sphere-ex.com/1_16c3bbe14b.png)\n\nShardingSphere-JDBC 方案\n\n目前 Apache ShardingSphere 由 JDBC、Proxy 和 Sidecar（规划中）这 3 款既能够独立部署，又支持混合部署配合使用的产品组成，在当当自研 WMS 中使用的是 ShardingSphere-JDBC。\n\n那么 ShardingSphere-JDBC 在 WMS 中具体是如何应用的呢？\n\n在 WMS 系统中，城市的“仓”就是仓储系统的管理单位，每个仓有对应的业务系统和数据库，单个仓使用 3 套 MySQL 主从集群承载指定城市仓库的数据。目前，当当在全国各地设有十余个自建的仓库，主要分布在订单量较大的城市，这种自建仓库模式在具备灵活管理等特性的同时，也可以降低企业的长期仓储成本。\n\n在架构规划上，WMS 通过 ShardingSphere-JDBC 按业务类型进行分库， 每套集群负责存储指定的业务数据，单个仓的 3 套 MySQL 集群分工如下：\n\n- 基础库集群：User，Area，Menu；\n\n- 业务库集群：Order，Package；\n\n- 库存库集群：Stock，Working。\n\n上线前，系统会根据仓库的基础数据进行初始化，如货位等信息，这里不做深入讨论。\n\n分布式数据库中间件的架构解决了 WMS 存算受限、开发维护成本高以及系统灵活度不足的问题，以成熟的底座和熟悉的技术栈去承载业务，架构相对透明化，兼容度良好且没有入侵性。\n\n![当当2.png](https://official-media.sphere-ex.com/2_783843d774.png)\n\n当当 WMS 系统拓扑图（单仓）\n\n\n## 用户收益\n\n在当当自研 WMS 的过程中发挥了非常重要的作用，带来了五方面的收益如下：\n\n1、性能极致\n\nShardingSphere-JDBC 轻量级的框架使其性能接近原生 JDBC，提供分库能力的同时性能可发挥极致，WMS 产能不再有受限的风险；\n\n2、系统稳定\n\n自 2016 年 WMS 上线至今，数年来系统一直稳定运行；\n\n3、低风险、零侵入\n\n当当底层系统为 2000 年演进而来，Apache ShardingSphere 凭借自身零侵入性的特点，用很小的改动就即完成了兼容适配，满足了业务需求；\n\n4、专注业务开发\n\n研发团队无需再关心业务拆分的工作，可将更多的精力聚焦于业务需求；\n\n5、降本增效\n\n高兼容的特点可以在无需大量的重构和改造前提下满足业务增长的需求，改造迁移成本低；     \n\n通过仓店智能化、入库商品自动匹配最优货位的设计，仓店订单效率提升了 30%，人力成本节约上千万。\n\n\n## Apache ShardingSphere 的前世今生\n\n有人说 ShardingSphere 是当当的产品，其实更准确的说：ShardingSphere 起源于当当，在 2018 年 11 月 10 日捐赠给 Apache，经 17 个月的孵化于 2020 年 4 月 15 毕业，是 2020 年度第一个从 Apache 孵化器毕业的顶级项目。\n\n前段时间，在 ShardingSphere 进入 Apache 基金会三周年的纪念日（2021/11/10）里，社区发布了期待已久的 5.0 正式版，借着这个机会，一起回顾 Apache ShardingSphere 的前世今生。\n\n- 2014 年，当当针对电商平台推出了统一开发框架 dd-frame，目的是通过分离业务和技术代码，实现开发框架统一化、技术组件标准化以及跨组沟通高效化，使研发人员的精力最大化的聚焦在业务开发上。开发框架中名为 dd-rdb 的关系型数据库模块专门用于负责数据访问，可实现简单的分库分表功能，它就是 Sharding-JDBC 的前身，也是 dd-frame 2.x 中重点规划的部分。     \n \n- 2015 年，当当决定重构 WMS 和 TMS，设计上需要引用分库分表方案，天时地利人和，当当架构部在 9 月正式启动了项目，同年 12 月份 Sharding-JDBC 1.0.0 在当当内部发布，正式开启了 ShardingSphere 的征途。\n \n- 2016 年初，Sharding-JDBC 从 dd-rdb 中剥离，向社区开源，产品形态是增强版 JDBC 驱动，以 jar 包形式提供服务。\n \n- 2017 年底，发布 2.0.0 版本，打开了数据库治理功能。\n\n- 2018 年，进入 Apache 软件基金会。3.0.0 版本是一个华丽的转身，独立服务形态的 Sharding-Proxy 面世，实现了异构开发语言的需求，同时项目名称从 Sharding-JDBC 更名为 ShardingSphere，迈出了构建数据库上层标准和生态的第一步。     \n \n- 2019 年，发布 4.0.0 版本，可支持更多的数据库产品。\n \n- 2020 年，毕业成为 Apache 软件基金会的顶级项目。    \n\n- 2021 年 11 月 10 日，发布 5.0.0 GA 版，作为加入 Apache 软件基金会三周年纪念日的礼物，呈现给社区及整个分布式数据库和安全生态领域。     \n\n![当当3.png](https://official-media.sphere-ex.com/3_ae2b522cd5.png)\n Apache ShardingSphere - RoadMap\n\n从 5.0.0 版本开始，Apache ShardingSphere 正式开启了以可插拔架构为核心的全新阶段，实现了由过去数据分片单一应用场景到现在复杂应用环境下综合数据治理的身份转变与能力提升。在新版本中，Apache ShardingSphere 具备了更多新特性，并拥有了更加成熟的大数据解决方案。\n\n首先，基于可插拔架构对内核进行了全面改造，内核中的各个功能可以任意组合并叠加使用。其次，为了提升 SQL 分布式查询能力，5.0.0 GA 版打造了全新的 Federation 执行引擎，来满足用户复杂的业务场景。此外，5.0.0 GA 版在内核功能 API 层面也进行了大量优化，旨在降低用户使用这些功能的成本。\n\n## 写在结尾\n\n全链条数字化驱动着当当高质量发展，让全民阅读触手可及。在必不可少的自研 WMS 一环中，ShardingSphere 在数据层提供了强力支撑。\n\nApache ShardingSphere 历经两年的迭代与打磨，5.0.0 GA 版本已面世。趋近成熟的可插拔生态，标志着 Apache ShardingSphere 从分库分表中间件工具蜕变为以 Database Plus 理念为新航标。重塑分布式可插拔体系，助力整个数据库行业带来新风向和更有价值的解决方案。\n\n更多内容欢迎点击阅读原文链接访问中文社区 OpenSEC（Open SphereEx Community），一起畅聊开源。\n\n","date":"2022-04-29","author":"SphereEx","excerpt":"历经两年的迭代与打磨，可插拔生态趋近成熟，标志着 Apache ShardingSphere 从分库分表中间件蜕变为以 Database Plus 为理念的分布式数据库生态系统。","metaDescription":null,"createdAt":"2022-04-29T08:49:24.970Z","updatedAt":"2022-11-02T05:36:20.351Z","publishedAt":"2022-04-29T08:49:28.304Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[{"id":14,"attributes":{"feature":false,"title":"Asia’s E-Commerce Giant Dangdang Increases Order Processing Speed by 30% — Saves Over Ten Million in Technology Budget with  Apache ShardingSphere","content":">Apache ShardingSphere is an easy-to-use and stable product, making Dangdang’s warehouse management system (WMS) even more powerful. Integrated with WMS, ShardingSphere plays a vital role in reforming the supply chain system.\n>-- Li Yong, Head of WMS Technology, Dangdang\n\nFollowing Apache ShardingSphere 5.0.0 GA release in November 2021, the 5.1.0 version was released last month. Having gone through over two years of polishing, ShardingSphere’s plugin-oriented ecosystem is beginning to take shape, and the project embarks on the evolution from a simple data sharding middleware to a mature distributed database ecosystem driven by the concept of Database Plus.\n\nDangdang, established at the end of 1999, has become a leading e-commerce platform selling books of any kind, and by integrating new Internet technologies with the traditional book industry. Dangdang was founded during the surge in China’s Internet industry in the early 2000s.\n\nLater, the e-commerce industry became exremely competitive in the country, and the market saturated. Facing fierce market competition, e-commerce platforms had to adapt to remain competitive.\n\nIn response, Dangdang not only adjusted its business strategies and management approaches, but also upgraded its technology architecture. Dangdang didn’t have its warehouse management and transportation system at that time. However, with growing business volume and technological capabilities, Dangdang needed to rebuild its warehouse management system and transportation management system (TMS) to better satisfy its business needs. For instance, in terms of hardware, it replaced mini-computer with x86, while its old centralized system was transformed to a distributed system with more flexibility.\n\nOne of the biggest challenges was massive warehousing data storage. The engineers wanted to adopt the data sharding technology that was often chosen by other big Internet companies. Disappointingly, they failed to find a mature and versatile open source database middleware in the marketplace, and therefore,started to develop a new data sharding product. That’s the origin of Sharding-JDBC. The product was created to bring more possibilities to data services.\n\nDangdang released its new WMS five years ago, which meant that it completed its intelligent warehousing transformation. Since then, with Apache ShardingSphere, the WMS has enabled Dangdang to hold large online shopping events every year such as Dangdang’s April Reading Festival, Double-Eleven Online Shopping Festival (aka Singles Day), and Mid-Year Shopping Festival, and to manage over a dozen of smart warehouses.\n\n## Business Challenges\n\nWhen Dangdang adopted a third-party WMS, the database in use was Oracle based on IBM minicomputers.\n\nHowever, considering the increasing business volume and warehouse order requests, especially during the online shopping festivals, the traditional centralized database architecture of the old WMS became inadequate because the computing and storage capabilities were limited. Additionally, the scale-up solution couldn’t support the system during online shopping festivals, and therefore the developers must do scale-up and adjust the business layer several times to alleviate the storage and computing limits and avoid production risks.\n\n- **Limited computing and storage capabilities**\n\nA centralized architecture is less scalable, making database computing and storage capabilities become the bottleneck.\n\n- **Expensive development and maintenance cost**\n\nBecause of poor scalability, developers have to make concessions and scale-up, which increases system development and maintenance costs.\n\n- **Exclusiveness**\n\nIf the architecture is not open enough, the system is less flexible, with fewer functions, and difficult to be transformed. The current architecture makes it difficult to quickly adopt new business services such as cloud native ones, SQL audit, data encryption, and distributed governance.\n\n## The Solution\n\nBased on the situation described above, Dangdang’s tech team proposed a warehouse management system solution: in terms of hardware, IBM minicomputer would be replaced with all-purpose x86, and MySQL would replace Oracle.\n\nHowever, at that time, there wasn’t a versatile and mature enough open source database middleware living up to Dangdang’s expectations, so they created one and named it Sharding-JDBC.\n\nShardingSphere-JDBC is positioned as a lightweight Java framework that provides additional services at the Java Database Connectivity (JDBC) layer. It is lightweight, efficient, easy to use, and compatible.\n\nWith ShardingSphere providing services in the form of the .jar files package, users can connect the client directly to the database without additional deployment and dependencies. It can be seen as an enhanced JDBC driver, fully compatible with JDBC and all ORM frameworks.\n\n- Compatible with JDBC and any JDBC-based ORM framework such as JPA, Hibernate, Mybatis, Spring JDBC Template.\n\n- Supports all third-party database connection pools such as DBCP, C3P0, BoneCP, HikariCP, etc.\n\n- Supports all databases implementing JDBC standards. Currently, ShardingSphere-JDBC supports MySQL, PostgreSQL, Oracle, SQL Server, and any database that can be accessed via JDBC.\n\n![1.jpeg](https://official-media.sphere-ex.com/1_3d03cbca69.jpeg)\nThe ShardingSphere-JDBC Topography\n\nCurrently, Apache ShardingSphere consists of three products, i.e. JDBC, Proxy, and Sidecar (TODO). ShardingSphere-JDBC and ShardingSphere-Proxy can be deployed independently or together.\n\nIt is ShardingSphere-JDBC that is used in Dangdang’s warehouse management system.\n\n\n>So how is ShardingSphere-JDBC exactly utilized?\n\nIn the warehouse management system, each warehouse positioned in a physical city is referred to as a unit with its corresponding business system and databases. Each warehouse has three sets of MySQL primary-secondary clusters to load the warehousing data of the designated city. So far, Dangdang has more than ten self-built warehouses all over China, mostly in cities where customers place a large number of orders. This self-built warehouse model is flexible for warehouse management and helps reduce storage costs in the long run.\n\nIn terms of architecture, the WMS uses ShardingSphere-JDBC to do database sharding according to their business types, and each cluster stores specified business data. The three MySQL clusters of a single warehouse are divided into three types as follows:\n\n-** Basic:** stores user, area, and menu data.\n- **Business:** stores order and package data.\n- **Inventory:** stores stock and working data.\n\nBefore the release, the system was initialized based on the basic data of a warehouse, such as storage locations.\n\nNext, by deploying the distributed database middleware, Dangdang successfully solved a series of problems such as limited storage and computing capabilities, high costs, and lack of flexibility.\n\n![1.jpeg](https://official-media.sphere-ex.com/1_b7199ae49f.jpeg)\nTopography of Dangdang’s Warehouse Management System (of a Single Warehouse)\n\n## User Advantages\n\nApache ShardingSphere played a significant role in helping Dangdang develop its WMS. There are five main benefits:\n\n- **Extraordinary performance**\n\nThe ShardingSphere-JDBC lightweight framework makes its performance close to a native JDBC. Apart from its great database sharding capability, it helps database performance be taken to the extreme, which allows the WMS to work at full capacity.\n\n- **Keep the system stable**\n\nWMS has been functioning well since its release in 2016.\n\n- **Low risk & zero invasion**\n\nThe underlying system of Dangdang has been evolving since 2000. Thanks to its zero-intrusion nature, Apache ShardingSphere can be compatible with others with small modifications to meet Dangdang’s business requirements.\n\n- **Allow developers to focus on the business side**\n\nThe developer team does not need to worry about sharding any more and can concentrate on developing the system to meet business needs.\n\n- **Cost effective and efficient**\n\nSince ShardingSphere is known for its high compatibility, to satisfy increasing business needs, developers don’t have to reconstruct or upgrade the system, minimizing the migration cost.\n\nWarehousing order processing speed is increased by 30% and accordingly, tens of millions of manpower costs are reduced due to the smart warehouses and the auto storage location matching technology.\n\n**Apache ShardingSphere’s Roadmap**\n\nSome said that ShardingSphere is a product created by Dangdang. To be precise, ShardingSphere was derived from the company and it also donated ShardingSphere to the Apache Software Foundation (ASF) on November 10, 2018.\n\nAfter 17-months in the ASF incubator, Apache ShardingSphere successfully graduated on April 15, 2020, as a Top-Level Apache Project.\n\nRecently, to celebrate the third anniversary of ShardingSphere entering Apache Software Foundation(ASF), the community released ShardingSphere 5.0.0. Below is a brief review of Apache ShardingSphere.\n\n- In 2014, Dangdang introduced a centralized development framework targeting at its e-commerce platform called dd-frame. It was created to unify the development framework, standardize its technical components, and achieve efficient cross-team communication by separating business code from technical code. In this way, engineers can devote all their efforts to the business side. The relational database module named dd-rdb in the framework was developed to handle data access and implement the data sharding function. It was the precursor of Sharding-JDBC, as well as a major part of dd-frame 2.x.\n\n- In 2015, Dangdang decided to rebuild its WMS and TMS. As it needed a data sharding plan, the team launched the project in September. In December, 2015, Sharding-JDBC 1.0.0 was released and used within Dangdang.\n\n- In early 2016, Sharding-JDBC was separated from dd-rdb and became open source. The product is an enhanced JDBC driver providing service in .jar files.\n\n- At the end of 2017, Version 2.0.0 was released with the new data governance function.\n\n- In 2018, ShardingSphere was enrolled into Apache Incubator. The release of Version 3.0.0 was a notable turnaround: Sharding-Proxy was released as an independent service. It supported heterogeneous languages, and the project was renamed from Sharding-JDBC to ShardingSphere. It’s in 2018 that the community decided to build the criteria and ecosystem above databases.\n\n- In 2019, Version 4.0.0 was released capable of supporting more database products.\n\n- In 2020, ShardingSphere graduated as a Top-Level Project of the ASF.\n\n- On November 10, 2021, Version 5.0.0 GA was released as a third-anniversity celebration with the whole Apache ShardingSphere community, and the distributed database industry.\n\n![1.jpeg](https://official-media.sphere-ex.com/1_83a06f3e37.jpeg)\nApache ShardingSphere—Roadmap\n\nSince Version 5.0.0, Apache ShardingSphere has embarked on its new journey: with the plugin oriented architect at its core, it evloved from a data sharding application to a comprehensive and enhanced data governance tool applicable to various complex application scenarios. Concurrently, Apache ShardingSphere also has more features, and big data solutions.\n\n## Conclusion\n\nDigitization motivated Dangdang to achieve high-quality development and fulfill its mission. ShardingSphere is glad to support Dangdang’s WMS with its cutting-edging data services.\n\nHaving gone through two years‘ development, Apache ShardingSphere 5.0.0 GA has been released. The pluggable ecosystem marks an evolution from a data sharding middleware tool to a pioneer in the industry following the “Database Plus” concept.","date":"2022-03-11","author":"SphereEx","excerpt":"Apache ShardingSphere is easy-to-use and stable, making Dangdang’s warehouse management system (WMS) even more powerful.","metaDescription":"Dangdang is a B2C-based e-commerce company in China offering a large selection of products both online and offline.","createdAt":"2022-04-29T10:52:10.330Z","updatedAt":"2022-09-28T02:42:43.896Z","publishedAt":"2022-04-29T10:52:12.009Z","locale":"en"}}]}}},{"id":40,"attributes":{"feature":true,"title":"基于Sharding-JDBC的订单分库⽅案","content":"# **前言** \n\n随着业务的发展，数据量迅速增⻓，单实例 Mysql 数据库的磁盘存储容量很快就会达到上限（阿⾥云 RDB 最⼤上限6T）。因此为了解决⽣产环境的存储瓶颈，需要对数据库进⾏分库设计，加⼤存储空间。**本⽅案基于Apache开源项⽬ ShardingSphere 采 Sharding-JDBC 进⾏订单⽔平分库设计（数据分⽚）缓解存储压⼒。**\n\n \n\n# **方案对比**\n\n⽬前主流的分库分表⽅案就是 Sharding-JDBC 和 Mycat，下⾯主要对⽐这两种。\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MGZlMTlmM2IyYTNiZDYzMTk5YjI0NWVlYjJlMzE2NmZfRXVzMURvRUdabmYxR1J4MTJxVWFkRGNuVE1GbjJia21fVG9rZW46Ym94Y25EMDBYVHBVUFhuRm51ZVpvS2Rja3ZjXzE2ODAwNzQ5MzU6MTY4MDA3ODUzNV9WNA)\n\n经上述对⽐，可以发现 Sharding-JDBC 是直连数据库、没有中间层的⽅式，性能上更优，同时由于 Sharding-JDBC 集成在应⽤代码内，并不会增加额外的运维成本，使开发者可以专注于⾃⾝代码逻辑。另外，Sharding-JDBC ⽬前仅⽀持 Java，考虑到⽬前团队技术栈主要以 Java 为主，所以采⽤ Sharding-JDBC 的分库⽅案成为了⼀个不错的选择。 下图反映了 Sharding-JDBC 在应⽤中所处的位置：\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=NTY1M2UyZjBmNWE1NThiMzg1YmI0YTczYmNmNDZhMjZfdHJtWU1pSUdNVnRLV3JabHo0cktNUndXbGxEWkdXYXhfVG9rZW46Ym94Y25XVGdhUGdOVFVid21ucHlTWnpFSFNkXzE2ODAwNzQ5MzU6MTY4MDA3ODUzNV9WNA)\n\n \n\n# **原理**\n\nSharding-JDBC 提供了精确分⽚扩展接⼝ org.apache.shardingsphere.api.sharding.standard.PreciseShardingAlgorithm. PreciseShardingAlgorithm，**只需要实现其doSharding ⽅法，基于分⽚键做⼀定的算法转换，返回指定的数据源，便可以达到分库的效果。**  ⽬前我们使⽤的是⼀致性 hash 算法，⽤于定位分⽚键所在数据源，同时引⼊虚拟节点，可以避免数据倾斜问题（即有的数据源数据多，有的数据源数据少，导致数据源设备压⼒不均衡）算法原理如图： \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=NWZkMDc1YmUxZTFmMzk5ZTg2ZThlYzhhM2NkZGU5MTRfa2ZzNDdsR3FTbnhJVTJaTENmRGFHMkZBMUN2S3hhN0pfVG9rZW46Ym94Y25qUWtZdWRYTmR0VG9SUHdtcTF5SFVnXzE2ODAwNzQ5MzU6MTY4MDA3ODUzNV9WNA)\n\n **然后基于该分库的分⽚键&分⽚算法，我们还需在 application.yml 中增加 sharding 数据源配置；同时需要增加分表相关的配置信息。**最后项⽬启动时，sharding 会⾃动加载这些配置初始化数据源。在请求时从请求参数中提取到分⽚键，应⽤分⽚算法，找到⽬标数据源，操作该数据源。 \n\n# **实战**\n\n在原有订单数据源 datasource_name 的基础上，新加3台订单数据库实例 datasource_name_1&datasource_name_2&datasource_name_3，扩充后共4台订单数据库实例，磁盘存储最⼤可以达到原有容量的4倍。\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=NjUzZGZlZDA4MmIwNWE4MTRhN2Y1MWIyZWFkNDc3NTVfcUZRMWFIMFNQWnNxR0UxTUVmc21uSmttRElTMjNETnBfVG9rZW46Ym94Y25GZmVJWWphbTJBY3lTOTdCSkZKNURnXzE2ODAwNzQ5MzU6MTY4MDA3ODUzNV9WNA)\n\n**4.1引⼊maven依赖**\n\n```xml\n<dependency>\n\t<groupId>org.apache.shardingsphere</groupId>\n\t<artifactId>sharding-jdbc-spring-boot-starter</artifactId>\n\t<version>4.1.1.001</version>\n</dependency>\n```\n\n这⾥的4.1.1.001版本是在官⽹ sharding 4.1.1版本上⾃⾏扩充了⼀部分内容：本⾝根据业务分⽚键对业务数据进⾏分⽚后，同⼀类型的数据只会在⼀个数据源中，sharding 只能操作这⼀个数据源的数据，但对于⼀部分分⽚键落⼊新数据源后，意味着数据源发⽣了变动，⽽在旧库数据未迁⼊到新库时，会存在⼀定的兼容时期，此时，改造源码实现了如下双操作（除插⼊外）：a、双查询：查询新旧数据源的数据，然后业务项⽬⾥使⽤切⾯进⾏查询汇总去重，根据 id+updateDatetime 进⾏去重，保留最新数据。b、双更新：更新数据时，会同时更新新旧数据源中满⾜条件的数据。c、双删除：删除数据时，会同时删除新旧数据源中满⾜条件的数据。双删的⽬的是，为了避免只 删除了⼀个库的数据，⽽由于双查导致另⼀个库的数据被查询出来误⽤。d、单插⼊：插⼊数据时，只会插⼊新分⽚算法所落⼊的数据源中。这样做的好处是，在不影响业 务的情况下，也减少了⼀部分数据迁移的⼯作量，即分库上线后新数据直接插⼊新库，不⽤做任何处理。\n\n改造源码如下：\n\n```java\npackage org.apache.shardingsphere.core.rule;\n\n/**\n * Databases and tables sharding rule.\n */\n@Getter\npublic class ShardingRule implements BaseRule {\n\n    // 定义：旧的分库分片策略\n    private final ShardingStrategy oldDatabaseShardingStrategy;\n   \n    public ShardingRule(final ShardingRuleConfiguration shardingRuleConfig, final Collection<String> dataSourceNames) {\n        Preconditions.checkArgument(null != shardingRuleConfig, \"ShardingRuleConfig cannot be null.\");\n        Preconditions.checkArgument(null != dataSourceNames && !dataSourceNames.isEmpty(), \"Data sources cannot be empty.\");\n        this.ruleConfiguration = shardingRuleConfig;\n        shardingDataSourceNames = new ShardingDataSourceNames(shardingRuleConfig, dataSourceNames);\n        tableRules = createTableRules(shardingRuleConfig);\n        broadcastTables = shardingRuleConfig.getBroadcastTables();\n        bindingTableRules = createBindingTableRules(shardingRuleConfig.getBindingTableGroups());\n        // 加载旧的分库分片策略\n        ShardingStrategyConfiguration oldDatabaseShardingStrategyConfig = shardingRuleConfig.getOldDatabaseShardingStrategyConfig();\n        if (oldDatabaseShardingStrategyConfig != null) {\n            oldDatabaseShardingStrategy = createDefaultShardingStrategy(oldDatabaseShardingStrategyConfig);\n        } else {\n            oldDatabaseShardingStrategy = null;\n        }\n        defaultDatabaseShardingStrategy = createDefaultShardingStrategy(shardingRuleConfig.getDefaultDatabaseShardingStrategyConfig());\n        defaultTableShardingStrategy = createDefaultShardingStrategy(shardingRuleConfig.getDefaultTableShardingStrategyConfig());\n        defaultShardingKeyGenerator = createDefaultKeyGenerator(shardingRuleConfig.getDefaultKeyGeneratorConfig());\n        masterSlaveRules = createMasterSlaveRules(shardingRuleConfig.getMasterSlaveRuleConfigs());\n        encryptRule = createEncryptRule(shardingRuleConfig.getEncryptRuleConfig());\n    }\n}\n```\n\n\n\n```java\npackage org.apache.shardingsphere.sharding.route.engine;\n\n/**\n * Sharding route decorator.\n */\npublic final class ShardingRouteDecorator implements RouteDecorator<ShardingRule> {\n    \n    @SuppressWarnings(\"unchecked\")\n    @Override\n    public RouteContext decorate(final RouteContext routeContext, final ShardingSphereMetaData metaData, final ShardingRule shardingRule, final ConfigurationProperties properties) {\n        SQLStatementContext sqlStatementContext = routeContext.getSqlStatementContext();\n        // SQL参数集合\n        List<Object> parameters = routeContext.getParameters();\n        ShardingStatementValidatorFactory.newInstance(\n                sqlStatementContext.getSqlStatement()).ifPresent(validator -> validator.validate(shardingRule, sqlStatementContext.getSqlStatement(), parameters));\n        // 获取分片条件\n        ShardingConditions shardingConditions = getShardingConditions(parameters, sqlStatementContext, metaData.getSchema(), shardingRule);\n        boolean needMergeShardingValues = isNeedMergeShardingValues(sqlStatementContext, shardingRule);\n        if (sqlStatementContext.getSqlStatement() instanceof DMLStatement && needMergeShardingValues) {\n            checkSubqueryShardingValues(sqlStatementContext, shardingRule, shardingConditions);\n            mergeShardingConditions(shardingConditions);\n        }\n        ShardingRouteEngine shardingRouteEngine = ShardingRouteEngineFactory.newInstance(shardingRule, metaData, sqlStatementContext, shardingConditions, properties);\n        RouteResult routeResult = null;\n        // ShardingStandardRoutingEngine特殊处理\n        if(shardingRouteEngine instanceof ShardingStandardRoutingEngine) {\n            SQLStatement sqlStatement = sqlStatementContext.getSqlStatement();\n            // 查询/更新/删除操作，开启双操作模式\n            boolean isCompatibleMode = (sqlStatement instanceof SelectStatement) || (sqlStatement instanceof UpdateStatement) || (sqlStatement instanceof DeleteStatement);\n            routeResult = shardingRouteEngine.route(shardingRule, isCompatibleMode);\n        } else {\n            routeResult = shardingRouteEngine.route(shardingRule);\n        }\n        if (needMergeShardingValues) {\n            Preconditions.checkState(1 == routeResult.getRouteUnits().size(), \"Must have one sharding with subquery.\");\n        }\n        return new RouteContext(sqlStatementContext, parameters, routeResult);\n    }\n}\n```\n\n\n\n```java\npackage org.apache.shardingsphere.sharding.route.engine.type.standard;\n\n/**\n * Sharding standard routing engine.\n */\npublic final class ShardingStandardRoutingEngine extends BaseShardingRoutingEngine {\n\n    /**\n     * 寻找sharding路由结点信息\n     * @param shardingRule\n     * @param tableRule\n     * @param databaseShardingValues\n     * @param tableShardingValues\n     * @param isCompatibleMode 兼容模式（即同时操作新旧分片算法定位到的数据库，查询/更新/删除自动开启兼容模式）\n     * @return\n     */\n    private Collection<DataNode> route0(final ShardingRule shardingRule, final TableRule tableRule, final List<RouteValue> databaseShardingValues, final List<RouteValue> tableShardingValues,\n                                        final boolean isCompatibleMode) {\n        Collection<DataNode> result = new LinkedList<>();\n        // 双查 & 双更 & 双删核心逻辑\n        // 1. 加载旧数据节点路由信息\n        ShardingStrategy oldDatabaseShardingStrategy = shardingRule.getOldDatabaseShardingStrategy();\n        Collection<String> oldDataSources = null;\n        if (oldDatabaseShardingStrategy != null && isCompatibleMode) {\n            oldDataSources = routeDataSources(shardingRule, tableRule, databaseShardingValues, true);\n            for (String each : oldDataSources) {\n                result.addAll(routeTables(shardingRule, tableRule, each, tableShardingValues, true));\n            }\n        }\n        // 2. 加载新数据节点路由信息\n        Collection<String> routedDataSources = routeDataSources(shardingRule, tableRule, databaseShardingValues, false);\n        for (String each : routedDataSources) {\n            // 新数据节点未被加载时，加载新数据节点\n            if(oldDataSources == null || !oldDataSources.contains(each)) {\n                result.addAll(routeTables(shardingRule, tableRule, each, tableShardingValues, false));\n            }\n        }\n        return result;\n    }\n    \n    private Collection<String> routeDataSources(final ShardingRule shardingRule, final TableRule tableRule, final List<RouteValue> databaseShardingValues, final boolean isOld) {\n        if (databaseShardingValues.isEmpty()) {\n            return tableRule.getActualDatasourceNames();\n        }\n        // 根据isOld参数，区分加载新/旧分片策略\n        Collection<String> result = new LinkedHashSet<>();\n        if (isOld) {\n            result.addAll(shardingRule.getOldDatabaseShardingStrategy().doSharding(tableRule.getActualDatasourceNames(), databaseShardingValues, this.properties));\n        } else {\n            result.addAll(shardingRule.getDatabaseShardingStrategy(tableRule).doSharding(tableRule.getActualDatasourceNames(), databaseShardingValues, this.properties));\n        }\n        Preconditions.checkState(!result.isEmpty(), \"no database route info\");\n        Preconditions.checkState(tableRule.getActualDatasourceNames().containsAll(result), \n                \"Some routed data sources do not belong to configured data sources. routed data sources: `%s`, configured data sources: `%s`\", result, tableRule.getActualDatasourceNames());\n        return result;\n    }\n\n}\n```\n\n **4.2数据分片**基于业务进⾏分库的分⽚键选择，最好能均匀的将数据拆分到不同数据库实例。本⽅案假设选择⽤⼾ID作为分库的分⽚键，然后⾃定义新的分库分⽚算法，**算法命名为 DatasourceShardingAlgorithmV2**，算法内容如下：\n\n```java\nimport org.apache.shardingsphere.api.sharding.standard.PreciseShardingAlgorithm;\nimport org.apache.shardingsphere.api.sharding.standard.PreciseShardingValue;\nimport java.nio.ByteBuffer;\nimport java.nio.ByteOrder;\nimport java.util.*;\n\n/**\n * 新的分库数据分片算法\n *\n * @author jingmin.yang\n * @date 2022/4/18 19:23\n */\npublic class DatasourceShardingAlgorithmV2 implements PreciseShardingAlgorithm<String> {\n    // 排序存储结构：SortedMap<虚拟节点，物理节点>\n    private static final SortedMap<Integer, String> virtualToRealMap = new TreeMap<Integer, String>();\n\n    /**\n     * 基于所有数据源初始化订单分库虚拟节点\n     * @return\n     */\n    static {\n        List<String> datasources = new ArrayList<String>();\n        datasources.add(\"datasource-name\");\n        datasources.add(\"datasource-name-1\");\n        datasources.add(\"datasource-name-2\");\n        datasources.add(\"datasource-name-3\");\n        // 初始化分库虚拟节点数据\n        init(virtualToRealMap, datasources);\n    }\n\n    @Override\n    public String doSharding(Collection<String> availableTargetNames, PreciseShardingValue<String> preciseShardingValue) {\n        // shardingKey即为分片键值\n        String shardingKey = preciseShardingValue.getValue();\n        return getShardingDatasource(virtualToRealMap, shardingKey);\n    }\n\n    /**\n     * 获取分库物理节点\n     * @param virtualToRealMap 分库虚拟节点\n     * @param shardingKey 分库的分片键标识\n     * @author: jingmin.yang\n     * @create: 2021/9/10 10:41\n     * @return\n     */\n    static String getShardingDatasource(SortedMap<Integer, String> virtualToRealMap, String shardingKey) {\n        int hashValue = hash(shardingKey);\n        // 顺时针找最近的虚拟节点\n        SortedMap<Integer, String> subVirtualToRealMap = virtualToRealMap.tailMap(hashValue);\n        if (subVirtualToRealMap.isEmpty()) {\n            return virtualToRealMap.get(virtualToRealMap.firstKey());\n        }\n        return subVirtualToRealMap.get(subVirtualToRealMap.firstKey());\n    }\n\n    /**\n     * 初始化hash环\n     *\n     * @param: virtualToRealMap\n     * @param: realNodes\n     * @author: jingmin.yang\n     * @create: 2021/9/14 16:29\n     */\n    static void init(SortedMap<Integer, String> virtualToRealMap, List<String> realNodes) {\n        for(String node: realNodes) {\n            virtualToRealMap.put(hash(node), node);\n\n            // 虚拟出指定数量的虚拟节点\n            int count = 0;\n            int i = 0;\n            int virtualNum = realNodes.size();\n            while (count < virtualNum) {\n                i++;\n                String virtualNode = node + \"#\" + i;\n                // 计算hash值\n                int hashValue = hash(virtualNode);\n                if (!virtualToRealMap.containsKey(hashValue)) {\n                    // 虚拟节点放到环上去 数据结构：红黑树\n                    virtualToRealMap.put(hashValue, node);\n                    count++;\n                }\n            }\n        }\n    }\n\n    /**\n     *\n     * MurmurHash是一种非加密型哈希函数，适用于一般的哈希检索操作,与其它流行的哈希函数相比，对于规律\n     * 性较强的key，MurmurHash的随机分布（离散性）特征表现更好，而且具有低碰撞率。Redis对节点进行\n     * shard时采用的是这种算法。\n     *\n     * @param: key 分片键\n     * @author: jingmin.yang\n     * @create: 2021/9/14 19:17\n     */\n    static int hash(String key) {\n        ByteBuffer buf = ByteBuffer.wrap(key.getBytes());\n        int seed = 305441741;\n        ByteOrder byteOrder = buf.order();\n        buf.order(ByteOrder.LITTLE_ENDIAN);\n        int m = 1540483477;\n        byte r = 24;\n\n        int h;\n        int k;\n        for (h = seed ^ buf.remaining(); buf.remaining() >= 4; h ^= k) {\n            k = buf.getInt();\n            k *= m;\n            k ^= k >>> r;\n            k *= m;\n            h *= m;\n        }\n\n        if (buf.remaining() > 0) {\n            ByteBuffer finish = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN);\n            finish.put(buf).rewind();\n            h ^= finish.getInt();\n            h *= m;\n        }\n\n        h ^= h >>> 13;\n        h *= m;\n        h ^= h >>> 15;\n        buf.order(byteOrder);\n        if (h < 0) {\n            h = Math.abs(h);\n        }\n        return h;\n    }\n}\n```\n\n测试： \n\n```java\npublic static void main(String[] args) {         \n    System.out.println(DatasourceShardingAlgorithmV2.getShardingDatasource(virtualToRealMap, \"U0000000001\"));\n}\n```\n\n结果：\n\n```Plaintext\ndatasource-name-1\n```\n\n **4.3规则配置**Sharding-JDBC 可以通过 Java, YAML, Spring 命名空间 和 Spring Boot Starter 这 4种⽅式进⾏配置，开发者可根据场景选择适合的配置⽅式。当前⽅案使⽤ YAML ⽅式进⾏规则配置，application.ym 内容如下（**红⾊部分为⾃⾏扩展内容**）： \n\n```yaml\nspring: \n shardingsphere:\n  # sharding数据源配置\n  datasource:\n    # 对应下方自定义的数据源名称\n    names: datasource-name,datasource-name-1,datasource-name-2,datasource-name-3\n    # 数据源基础配置\n    datasource-name:\n     # 数据源驱动类型\n      type: com.alibaba.druid.pool.DruidDataSource\n      # 数据库驱动类\n      driver-class-name: com.mysql.jdbc.Driver\n      # 数据库连接地址\n      url: jdbc:mysql://127.0.0.1:3306/datasource_name_0\n     # 数据库用户名\n      username: username\n     # 数据库用户密码\n      password: password\n    # 自定义第二个数据源（新数据源1）\n    datasource-name-1:\n      type: com.alibaba.druid.pool.DruidDataSource\n      driver-class-name: com.mysql.jdbc.Driver\n      url: jdbc:mysql://127.0.0.1:3306/datasource_name_1\n      username: username\n      password: password\n    # 自定义第三个数据源（新数据源2）\n    datasource-name-2:\n      type: com.alibaba.druid.pool.DruidDataSource\n      driver-class-name: com.mysql.jdbc.Driver\n      url: jdbc:mysql://127.0.0.1:3306/datasource_name_2\n      username: username\n      password: password\n    # 自定义第四个数据源（新数据源3）\n    datasource-name-3:\n      type: com.alibaba.druid.pool.DruidDataSource\n      driver-class-name: com.mysql.jdbc.Driver\n      url: jdbc:mysql://127.0.0.1:3306/datasource_name_3\n      username: username\n      password: password\n  # 采用sharding-jdbc进行分库分表设计\n  sharding:\n    # 默认数据源\n    default-data-source-name: datasource-name\n    # oldDatabaseStrategy是自行扩展配置（可选），用于双操作（双查询&双更新&双删除）中获取原有数据源，pom需要对应引入4.1.1.001自行扩展版本\n    oldDatabaseStrategy:\n      standard:\n        # 分库分片键\n        shardingColumn: sharding_key\n        # 原有的数据源分片算法，本算法返回的是固定的原有数据源datasource-name\n        preciseAlgorithmClassName: com.yang.sharding.DatasourceShardingAlgorithm\n   # 订单分库新的分片算法，根据sharding_key进行分库\n   default-database-strategy:\n      standard:\n        # 分库分片键\n        shardingColumn: sharding_key\n        # 新的分库分片算法\n        preciseAlgorithmClassName: com.yang.sharding.DatasourceShardingAlgorithmV2\n    # 分库分表配置，只有分库&分表的表需要在此配置，不分库不分表不需要配置，数据会存入默认数据源\n    tables:\n      # table_0使用分库+分表\n      table_0:\n        actual-data-nodes: datasource-name.table_0_$->{0..63},datasource-name-$->{1..3}.table_0_$->{0..63}\n        logic-table: table_0\n        # 数据库表策略配置\n        table-strategy:\n          standard:\n            # 表分片键\n            shardingColumn: sharding_key\n            # 表分片算法\n            preciseAlgorithmClassName: com.yang.TableShardingAlgorithm\n      # table_1使用分库，不使用分表\n      table_1:\n        actual-data-nodes: datasource-name.table_1, datasource-name-$->{1..3}.table_1\n        logic-table: table_1\n```\n\n**4.4代码改造准备**由于 sharding 是对分⽚键使⽤分⽚算法，获取到对应的数据源进⾏操作，⽽分⽚键是从请求 sql 中提取⽽来，所以需要在代码层⾯改造⽀持分库，所有 CRUD 操作必须要带上分⽚键参数。\n\n```sql\n<!-- 这里假设数据库是MySQL，分片键是user_id字段 -->\n<!-- 新增操作 -->\nINSERT INTO order_info('id', 'user_id', 'update_time') VALUES('O0000000001', 'U0000000001', NOW());\n<!-- 查询操作 -->\nSELECT * FROM order_info WHERE user_id='U0000000001';\n<!-- 更新操作 -->\nUPDATE order_info SET update_time=NOW() WHERE user_id='U0000000001';\n<!-- 删除操作 -->\nDELETE FROM order_info WHERE user_id='U0000000001';\n```\n\n \n\n# **总结**\n\n**⾄此，基于 Sharding-JDBC 完整的订单分库改造完成，包含选择分库分⽚键、定义分库数据分⽚算法，添加数据源&表配置，代码改造等。**本⽅案已有实际应⽤案例，如有需要的⼩伙伴可作为参考，有问题也可直接联系我，⼀起讨论学习。","date":"2022-04-24","author":"杨敬敏 Ginee后端工程师","excerpt":"本⽅案基于Apache开源项⽬ ShardingSphere 采 Sharding-JDBC 进⾏订单⽔平分库设计（数据分⽚）缓解存储压⼒。","metaDescription":null,"createdAt":"2023-03-29T07:36:03.794Z","updatedAt":"2023-03-29T07:36:04.607Z","publishedAt":"2023-03-29T07:36:04.603Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":38,"attributes":{"feature":true,"title":"金鹰系统数据库分库分表策略","content":"为了全面监测区域内企业的金融风险，金鹰系统汇集了工商、舆情、投诉举报等多维度的大量数据。随着时间的推移，积攒的数据量越来越大，对系统的数据处理效率提出了更高的要求。目前金鹰系统单表最大数据量已达到亿级别，传统的数据库性能优化手段已无法满足系统的性能需求。为了减少数据库的负担，提升数据库响应速度，缩短查询时间，防控中心技术团队对数据进行分库分表。\n\n**什么是分库分表？**\n\n分库指从单个数据库拆分成多个数据库的过程，将数据散落在多个数据库中。而分表指从单张表拆分成多张表的过程，将数据散落在多张表内。\n\n**分库分表能解决什么问题？**\n\n如果数据库的查询压力过大，就需要考虑拆库，通过分库来分担单个数据库的连接压力。比如，数据库每秒需处理3500笔请求，假设单库每秒能处理的请求数最大为1000笔，那么就可以考虑拆分成4个库，来分散查询连接压力。\n\n如果单表数据量过大，当数据量超过一定量级后，无论是对于数据查询还是数据更新，在经过索引优化等纯数据库层面的传统优化手段之后，还是可能存在性能问题。这时候就需要去换个思路来解决问题。比如：从数据生产源头、数据处理源头来解决问题，既然数据量很大，那我们就来个分而治之，化整为零。可以通过分表把数据按照一定的规则拆分成多张表，来解决单表环境下无法解决的存取性能问题。\n\n►►►\n\n**金鹰系统数据库分库分表策略**\n\n金鹰系统目前采用ShardingSphere（一款分库分表框架）作为数据库中间件。通过对企业名称、法人等多个关键字进行组合散列运算得到企业ID。以企业ID作为分片键，ShardingSphere将根据企业ID与配置的路由信息决定该企业的数据落于哪个数据库分片、哪张表上，合理的算法使数据均匀分布在各数据库分片上，不同分片保存的数据量大致相同，从而确保数据处理效果能得到最大的优化。 \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MGM5YWQ1NmU0MzBlNzFiYzhmMjYyN2MyNjk5MTE5ZThfQzlkaERYQzZWTWVWRFBkNkRYTm1WcG13eWtlZXVDZzVfVG9rZW46Ym94Y25QSGdXUEwwaTQwenB6eWRoajFIU2xlXzE2ODAwNzM5MzQ6MTY4MDA3NzUzNF9WNA)","date":"2022-04-21","author":"广州金融科技","excerpt":"金鹰系统通过对监管企业的海量信息数据采用数据分库分表方式治理，提高了数据查询、数据处理效率","metaDescription":null,"createdAt":"2023-03-29T07:12:54.670Z","updatedAt":"2023-03-29T07:12:56.574Z","publishedAt":"2023-03-29T07:12:56.571Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":42,"attributes":{"feature":true,"title":"智能电站数据中台建设与应用","content":"## **引言**\n\n \n\n近几年随着计算机技术、通信技术、人工智能技术的发展，电站正逐步向网络化、数字化、智能化迈进。智能电站是以电站生产和管理活动中各种信息的数字化为前提，计算机、网络、通信技术为基础，通过采用数据分析和处理、智能控制、自动决策等技术，使得在各种环境条件下都能够提供经济、环保、安全、稳定的电能并与智能电网高度融合的发电站([1-3])。数据信息资源正逐步成为电站的关键要素资源。由于电站信息系统大多是分阶段、分功能实施的，在电站智能化建设过程中，特别是在数据资源使用过程中暴露出来了许多问题。（1）数据孤岛现象严重。各个业务系统数据各自存储、各自定义，数据无法有效连接，数据分散、指标重复建设、数据实用价值低的现象普遍存在。（2）数据缺乏全生命周期管理。电站设计、基建阶段产生的数据不能传递到生产、运维阶段，影响了后期机组及设备的性能诊断和运行优化分析。（3）数据能力未能积累和沉淀。数据服务化程度低，数据共享和交互上缺乏整体协同性，无法形成有效的电站数据资产框架。\n\n如何唤醒电站沉睡的数据资产、打通数据信息壁垒、消除数据烟囱，把数据融合利用起来，是电站智能化转型过程中的难题。而以构建数据资产体系、释放数据资产价值为核心的数据中台正是这一问题的完美解决方案，对提升电站精益化运行管理水平具有十分重要的意义。\n\n \n\n### **1  数据中台架构设计**\n\n \n\n**1.1  电站数据源**\n\n来自电站不同设备和系统的数据主要是生产数据和管理数据，数据类型包含结构化、半结构化和非结构化3类数据。（1）结构化数据包括时序数据和非时序数据。时序数据一般指控制系统现场传感器数据采集过来的数据，比如火电厂的锅炉设备温度、压力等传感器信号，经过规整后存入工业实时数据库中；非时序数据包括各类关键业务系统数据，如智能检修、智能燃料、智能运行、智能经营等系统数据，这些数据基于关系型数据库存储，表现为二维形式。（2）半结构化数据包括各类系统的配置文件、产生的日志文件等，这类数据一般以XML、JSON或自定义文本文件格式存储。（3）非结构化数据包括电站档案系统、门禁系统、智能摄像头、移动机器人产生的各类技术文档、图像文件、音频文件、视频文件等。电站现有平台的管理数据与生产数据没有有效融合，实时数据库与关系数据库的互联互通也存在一定问题，未能进行数据有效组织管理和数据赋能。\n\n**1.2  数据中台架构**\n\n智能电站数据中台通过构建面向电站应用的数据共享和数据分析智能平台，克服了电站现有系统的技术缺陷。数据中台以数据资产高效管理为基础，沉淀数据通用能力，通过构建规范的、安全的、智能的、全域化的数据处理架构，提供数据采集、数据存储、数据萃取、数据治理、数据服务、数据可视化等全链路一体化的服务([4-9])，其技术架构如图1所示。\n\n \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ZGEyOTI5OGNmOGY2OGIwZDJmZjM1MjE0MTU3NGE2ZDFfUEgxamM0bWJEWFY4UmllVWNhUktLWVc5SW0xeHVoR1BfVG9rZW46Ym94Y25HYkllUWh2WERZWGtwY25kMWwzNUJmXzE2ODAxNTc3MDQ6MTY4MDE2MTMwNF9WNA)\n\n<center>图1  智能电站数据中台技术架构</center>\n\n<center>Fig.1  Technology architecture of intelligent power station data middle platform</center>\n\n数据采集是数据中台接入数据的入口，把各种异构数据源的数据采集到数据中台进行集中存储；数据萃取与分析将低价值的原始业务数据转换为高价值的数据资产，通过数据资产框架进行治理；基于数据服务体系将数据以服务化方式交付给业务系统，通过数据可视化框架进行数据展示，激活整个数据中台。\n\n**1.3  数据中台价值**\n\n数据中台的价值体现在“存、通、用、统、快”，如图2所示。通过构建智能电站数据中台，打通数据隔阂，统一数据标准，实现一切业务数据化、一切数据业务化、快速响应用户需求。（1）一切业务数据化。通过业务信息系统的建设，将系统中沉淀的业务数据汇聚到数据中台进行存储，实现一切业务数据化。（2）一切数据业务化。数据中台基于业务系统产生的数据进行体系化加工，将结果以服务化方式提供给业务使用，进而反哺业务，实现一切数据业务化。 \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=Y2Y0OGY1YmU1MzhkNzljNTAxNDg2MTM1ZWQxODQ1YmJfRVZkUnNSbXNjQWpXQmN2aFpZU0NRaTJLR1ZpOE9HcFRfVG9rZW46Ym94Y25QR2tWS3JRZzExcVZkaG5BVkpHdWljXzE2ODAxNTc3MDQ6MTY4MDE2MTMwNF9WNA)\n\n<center>图2  数据中台价值</center>\n\n<center>Fig.2  Value of data middle platform</center>\n\n \n\n### **2  数据中台关键技术**\n\n \n\n**2.1  全域数据采集**\n\n全域数据采集是智能电站数据中台建设的第一步，也是非常重要的一步，为数据中台全面汇集异构数据。数据采集平台可以针对不同的数据源，建立不同的数据采集任务，具备数据采集、转换、传输、复制、迁移、同步等能力([10-11])。（1）电站每台机组控制系统大约有10万个传感器测点，时间戳精度为秒级。转子设备振动数据时间戳精度可以达到毫秒级。采用自主研发的实时数据采集传输程序，基于例外发送、数据缓存、断点续传和历史回补机制，实现电站实时数据的快速高效传输，目前支持OPC、Modbus、101~104规约、CDT规约、RTU规约及其他私有TCP/UDP通信协议。（2）电站业务关系数据库众多，其采集与同步可使用Sqoop和Kettle开源程序。抽取模式可采用全量抽取和增量抽取模式，增量数据抽取一般基于时间戳、触发器、更新标识等机制。（3）电站日志系统数据采集可采用Flume。（4）针对电站外部互联网数据，使用网页爬虫方式自动抓取目标数据。\n\n**2.2  数据湖底座建设**\n\n数据湖底座是智能电站全域数据的存储中心，针对不同数据源和不同数据应用分析需求提供灵活、可扩展的混合型存储架构，以分布式存储系统为核心，能够实现任意规模结构化、半结构化和非结构化数据存储，以提高生产力、主动维护设备，为电站的数据融合提供支撑，为智能运行、智能检修等生产决策提供依据。（1）电站机组控制系统的传感器测点数据采集频率和时间戳精度均为秒级，考虑到存储性能和后期查询、统计、分析性能，采用自主研发的睿腾（Rython）工业实时数据库进行存储，数据库系统架构如图3所示。睿腾实时数据库基于高并发网络通信技术、IO资源管控技术、多级高速数据缓存算法、无损压缩和有损压缩相结合的机制，实现海量测点数据的高效传输、存储、管理和访问。 \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=YTVlYjFlN2MxZTE2NWRhZjU1YWU3NjY2ODgyMDY0MjlfTHkxRFhNeGRhbTAwYWQ1SEF0MTBsSjV2SzJnZnhtNWtfVG9rZW46Ym94Y25uVm40WElHdHRiUHJmbFo3bU9kanVnXzE2ODAxNTc3MDQ6MTY4MDE2MTMwNF9WNA)\n\n<center>图3  睿腾实时数据库系统架构</center>\n\n<center>Fig.3  System architecture of Rython real-time database</center>\n\n \n\n睿腾实时数据库可管控的测点规模可达千万级别。测点表基于两级索引构建，其索引结构如图4所示。在此索引结构中，如果已知一个测点的记录号RecNo，就可通过以下步骤在测点表文件中进行索引定位并进行相应的操作：①计算RecNo/10240可得到该测点所属的一级索引表单元位置，并从一级索引表中读取到二级索引块的起始文件块号；②计算RecNo % 10240可得到该测点所属的二级索引块的单元位置，并从二级索引块中读取到测点属性的文件块号；③根据索引得到的测点属性文件块号，可对该测点属性进行创建和编辑操作；如果需要删除一个标签，则只需要将该测点的二级索引块单元中的标签属性文件块号的高位（第31Bits）置为1即可，不需要更新测点属性。\n\n \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ZWI2NzQ5ZjY2ZGFiYjBlYTUzOTZkYjA5NzgwYzZjODVfbVNlZEI3V2tsZ2VVelROYThBeU9NNURZVldlalhweWdfVG9rZW46Ym94Y25aVTE4eVdCZGthSkV0aTEyYTVhN0tjXzE2ODAxNTc3MDQ6MTY4MDE2MTMwNF9WNA)\n\n<center>图4  睿腾实时数据库测点表索引结构</center>\n\n<center>Fig.4  Tag’s index structure of Rython real-time database</center>\n\n（2）分布式大数据平台基于自主研发的大数据产品，以分布式文件系统为核心，由管理节点和一定数量的工作节点组成大数据存储集群，支撑大数据分析、萃取、挖掘，保存加工处理后的结构化数据和图片、文档等非结构化数据([12])。（3）电站关键业务数据采用分布式关系数据库进行存储。关系数据库平台基于改进的MySql+MHA的部署方案，提供高可用、高数据吞吐能力、扩展性强的关系型数据库引擎。对于分布式数据访问层，采用开源shardingjdbc技术路线。\n\n**2.3  数据萃取与分析**\n\n基于数据开发和算法开发平台，数据萃取层通过电站业务主题域和数据仓库建设，构建完整的智能电站测点数据体系、指标数据体系、设备数据体系、成本数据体系、人员数据体系和文档数据体系。数据开发是将价值不大的原始数据变成对业务有价值的数据资产。数据开发平台产品提供批量计算、流式计算和查询分析能力([13])。（1）批量计算基于MapReduce、Spark等计算框架进行处理，主要用于人机交互较少的批量数据高延时处理，如离线数据仓库的加工、大规模数据清洗与挖掘等，其数据吞吐量大、延时高。（2）流式计算基于Storm、Spark Streaming等计算框架进行处理，数据源为源源不断的流式数据，适用于对数据的加工处理和应用有较高时效性的场景，如设备异常监控告警。（3）查询分析使用ElasticSearch、Kylin、ClickHouse等技术进行处理，适用于历史数据的在线查询、条件过滤和维度分析。算法开发作为一站式的企业机器学习开发平台，包括数据集管理（如数据接入与标注等）、核心算法组件（如回归、聚类、神经网络等）编写、主流算法框架（如TensorFlow、PyTorch、Mahout等）集成、可视化建模，旨在为企业快速提供核心算法服务([14])。基于自主研发的分布式图形化建模计算平台，通过简单的拖拽操作，就可组态出复杂的算法逻辑，为数据中台提供强大的数据建模和计算能力。图形化建模计算平台主要包含以下功能。（1）模型设计器：提供可视化的机理建模、数学建模工具。（2）工程组态：组态工程的编辑、调试、上线、维护管理。（3）算法因子：提供基础通用算法、热力学专用算法、大数据算法等不同种类算法因子。同时可以自定义算法，具备集成第三方算法的能力。（4）计算引擎：基于时间、条件、公式等基础引擎，采用流式计算、离线计算、内存计算等计算模式，进行分布式在线和离线计算。\n\n**2.4  数据资产治理**\n\n数据治理框架，为智能电站提供统一数据标准制定、主数据管理、元数据管理、数据质量管理等能力，通过数据地图和数据血缘实现电站数据360°全链路追踪。框架提供数据全生命周期的管控机制，当发生数据质量问题时，可通过可视化图形方式追溯指标的血缘及逻辑关系、关联指标的数据和权重，进而定位问题([15])。（1）电站资产模型通过设备树的方式进行组织定义，每个设备都用元素表示，元素的关联数据将作为元素的属性进行存储，元素属性数据包括常量、实时数据库测点值、关系数据库引用值、指标公式计算值、图片、文档等。（2）数据治理计算框架基于分布式消息引擎RocketMQ和分布式缓存Redis集群，采用事件触发和周期触发机制，实现设备树海量元素属性数据的层次计算和分析计算。**2.5  数据服务化交付**智能电站数据中台将数据进行服务化交付，企业内部业务系统和第三方应用不必关心底层数据库结构，就可以获得目标数据。通过这样的架构设计，对下屏蔽了异构数据的差异性，对上提供了统一的数据服务接口和标准化数据。当新的业务需求出现时，可以快速进行开发响应。基于自主研发的数据服务网关，可以为业务应用提供以下3种形式的数据服务。（1）OPC UA数据服务。在应用端对数据访问效率要求极高时，如实时性很强的测点数据，推荐使用OPC UA服务。（2）Restful WebAPI数据服务。基于微服务架构和SpringCloud技术体系，提供基于HTTP/HTTP2的服务接口。（3）文件数据服务。基于FastDFS分布式文件系统，为业务应用提供在线文档服务。\n\n**2.6  数据可视化框架**\n\n数据可视化框架基于Vuejs前端开源javascript库和CSS响应式布局模式构建，框架设计器如图5所示。框架组件设计采用MVVM（model view viewmodel）模式进行定义，框架内部基于HTML5、Canvas和WebGL图形绘制技术，实现了大量的2D、3D、VR等图形组件。框架组件间消息通讯采用Weak Event机制实现，这样不仅降低了组件间的业务耦合度，也消除了内存泄漏的风险。开发人员或业务人员使用数据可视化框架能够快速进行生产运营决策驾驶舱的数据准备、指标集建立、业务模型配置、管理视图及可视化图表构建，实现对电站海量数据的汇总统计、多维分析、报表呈现、图形展示，以交互、生动的形式直观、全面、快速地了解电厂生产经营情况并进行辅助决策，进而提升工作效率。 \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=Mjg5OGFlZWUyNTljOTYxMWJjZGRjMWQ1NDhmNGNmNDlfTUhaazU0VTFvT1d6WWt5RjcxNDJVOUtaUGk1R1VTNHFfVG9rZW46Ym94Y25XaUJJUGJNRWdYa1pDOGZJVWkyODliXzE2ODAxNTc3MDQ6MTY4MDE2MTMwNF9WNA)\n\n<center>图5  数据可视化框架设计器</center>\n\n<center>Fig.5  Designer of data visualization framework</center>\n\n \n\n### **3  数据中台应用**\n\n \n\n建设智能电站数据中台的最终目的是高效组织利用数据资源，提升电站精益化运行管理水平。以智能电站建设过程中的3个典型应用场景为例，阐述企业如何应用数据中台，提高数据治理、分析计算和展示能力。\n\n**3.1  设备运行监视**\n\n电站在运行过程中，需要对锅炉、汽轮机、发电机、磨煤机、引风机等关键主辅机设备进行实时监控。设备数据种类复杂、来源众多，以磨煤机为例，主要监视的信息如下。（1）一次测点数据。这些数据来自现场控制系统传感器，如磨煤机电流、功率、进口风量、出口风粉温度等，采集频率为秒级，为时间序列型数据。（2）指标数据。这些数据由不同业务系统产生，如发电量、利用小时、非计划停运次数等指标来自可靠性系统，而发电量、负荷率、耗煤量等指标来自在线经营系统。指标数据均通过一次测点数据不同的运算逻辑统计计算得到。（3）设备基础信息。这些数据主要来自设备管理系统，主要包括设备名称、编码、设备KKS编码、所属专业、设备分类、型号规格、生产厂家、出厂编号、安装地点、安装现场图片等信息。（4）设备检修和缺陷记录。这些信息主要来自ERP系统（如SAP），包括检修发生时间、故障描述、参与人员、耗费人工、使用物料、消缺状态等。在传统信息系统建设方式下，电站的各种系统大多是独立建设的，存在指标重复计算、统计口径不一致、数据信息无法互联互通等问题，也无法很好地应对快速变化的前端业务。通过建设智能电站数据中台，可以消除数据壁垒、统一数据标准，实现异构数据融合和设备运行监视的数据可视化快速配置展示。\n\n**3.2  脱硫优化系统**\n\n当前环保压力日益增大，在保证烟气二氧化硫排放达标的前提下，实现脱硫系统的安全、经济、优化运行日益重要。脱硫优化系统基于智能电站数据中台，采用数据驱动模式和大据挖掘技术，进行机组工况的规律发现、特征抽取、自动识别和智能寻优，给运行人员提供及时精细的操作指导，实现企业效益的最大化([16-18])。（1）确定优化关联指标。对脱硫系统的历史运行原始数据进行多维相关性分析，确定与寻优目标“浆液量”强相关的指标：机组负荷、入口二氧化硫浓度、入口烟气流量、入口烟气温度、入口烟气湿度、吸收塔浆液pH值和出口二氧化硫浓度。（2）采集机组稳定运行工况。一般机组负荷波动不超过15%、至少持续运行1 h认为是一个稳定工况。基于机组运行的海量数据，在数据中台萃取层配置提取规则，进行脱硫系统优化关联指标数据的采集和统计，存储到中台数据湖底座。（3）工况标签化分类。采用中台提供的聚类算法对不同负荷工况进行分类，并对聚类类心进行合理选择。（4）以聚类获得的类心作为训练样本，采用中台提供的有监督机器学习算法建模，如图6所示，建立脱硫系统优化模型([19-21])。 \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ZTI2NzRmNjIxYjViYWJhOWJhYWQ2ZDRjZGY4Yzk2ZTdfcUp4S1VMV0psektTeVJHMjBwZTNjMTdwcG9Sd3EwVzBfVG9rZW46Ym94Y25VWHp0eWpyVjRFNkM4Y2dyZ3lwTEFlXzE2ODAxNTc3MDQ6MTY4MDE2MTMwNF9WNA)\n\n<center>图6  脱硫系统优化机器学习模型</center>\n\n<center>Fig.6  Machine learning model of desulfurization system optimization</center>\n\n（5）根据实时采集的机组负荷、入口二氧化硫浓度、入口烟气流量、入口烟气温度、入口烟气湿度、吸收塔浆液pH值和出口二氧化硫浓度数据，与稳定工况进行匹配，获得期望的“浆液量”优化目标值。\n\n**3.3  大屏幕和移动应用发布**\n\n大数据时代，电站业务数据场景分析和终端展示呈多样化发展。企业通过大屏幕发布系统，可以建立良好的形象宣传窗口；通过终端移动应用，可以随时随地查询生产实时数据、浏览生产过程图形、监控主要设备状态，对关键运营指标进行分析、钻取和统计分析。基于中台数据治理和可视化框架，采用海量酷炫图表组件和所见即所得数据源配置，用户可以快速进行大屏幕和移动应用功能设计。图7展示了智能电站“发电量分析”移动应用业务场景。 \n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MmZiY2E1OGUxYjNjODliNjNmNWRiMTNjOTI4OTE5YTRfczBFWnhVRVVweGRBcXJTVzIwcWg2TzhYRXhzTVdjYkpfVG9rZW46Ym94Y25yZzV1UU5kZktJRmlHRWh2czdTY2xsXzE2ODAxNTc3MDQ6MTY4MDE2MTMwNF9WNA)\n\n<center>图7  智能电站移动应用</center>\n\n<center>Fig.7  Mobile app of intelligent power station</center>\n\n \n\n### **4  结语**\n\n \n\n（1）针对电站现有系统存在的数据孤岛、数据未能有效利用问题，分析了电站发电过程中产生的海量数据的数据源类型，设计了智能电站数据中台架构，消除了数据烟囱，统一了数据标准，实现了多源异构数据融合。（2）以数据资产高效治理为基础，通过全域数据采集、数据湖底座建设、数据萃取与分析、数据服务化交付、数据可视化等关键技术方案设计，构建和实现了智能电站数据中台。（3）介绍了智能电站数据中台的典型应用场景，阐述了如何应用数据中台提高数据治理、分析计算和展示能力，对于发电企业建设和应用数据中台具有一定的借鉴意义。","date":"2022-04-20","author":"王毅, 王智微, 何新","excerpt":"在电站智能化建设过程中，为解决现有系统的数据孤岛、数据利用率低等问题，设计了数据中台产品。基于全域数据采集、数据湖底座建设、数据萃取与分析、数据服务化交付、数据可视化关键技术，建立了智能电站数据中台，实现了多源异构数据融合、共享和服务能力，有效提升电站精益化运行管理水平。最后介绍了智能电站数据中台的典型应用场景，对发电企业建设和应用数据中台具有一定的参考价值。","metaDescription":null,"createdAt":"2023-03-30T06:40:05.006Z","updatedAt":"2023-03-30T06:41:58.790Z","publishedAt":"2023-03-30T06:41:58.787Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":43,"attributes":{"feature":true,"title":"OPPO云数据库访问服务技术揭秘","content":"## **1、背景**\n\nMySQL是OPPO使用最广泛的关系数据库，不同编程语言的微服务都是通过MySQL官方的SDK直连真实的数据库实例。这种最传统的使用方式，会给业务开发和数据库运维带来一系列影响效率和稳定性的问题。\n\n- 不合理的数据库访问，比如不恰当的连接池设置、高危SQL无拦截、无限流和熔断等治理能力\n- 无弹性伸缩能力，单机数据库性能或者容量不足时，扩容非常繁琐低效\n- 缺乏常用的功能特性，不支持读写分离、影子库表、单元化架构、数据库加密\n- 不支持跨语言，并且与应用服务强耦合，编码和升级都非常困难\n\n以上问题，我们讲通过CDAS来解决单MySQL带来的一系列问题。\n\n## **2、CDAS产品简介**\n\nMySQL在并发能力、稳定性、高可用方面久经考验，是绝大多数网联网产品首选的OLTP场景存储组件，但单机MySQL在超高并发、海量存储、OLAP能力上先天不足，因此当数据到达一定量级后一般采用分库分表的方式来水平扩展MySQL，提升系统的整体处理能力。CDAS基于分片理念设计，目的是解决MySQL单机性能瓶颈，对外提供超过并发、海量存储、支持HTAP的MySQL服务。在一组MySQL集群前搭建一套高可用的代理 + 计算集群，提供分片能力、自动化的弹性伸缩能力、读写分离、影子库、数据加密存储能力，为用户提供一体化的产品。\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ZjhkYTExOTEyN2EwNjBlYzE3YjIzZjEzZDRhMGYyMTZfM3dFcXZ0d1RCN2FjVGVxeXpYbWJ2ZTJnNGFuNGFRd1VfVG9rZW46Ym94Y25tWlhpYU85Wjcxb2RVM2NEUlF0SXNiXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\nCDAS Proxy使用Java语言开发，基于开源产品Apache ShardingSphere Proxy改造而来，并添加了许多高级特性来支持内部业务。我们的开发理念是依托开源的成熟产品来搭建服务，从社区汲取营养的同时也把发现的问题、特性回馈给社区，积极参与、和社区共同发展。 \n\n**2.1 CDAS产品特点**\n\n**（1）稳定性**\n\nCDAS进行了大量的基准测试、调优，实现了特殊的动态队列 + 线程池模型来保证Proxy的总体并发程度不会由于线程数过多导致上下文切换频繁从而性能下降，同时Proxy在长时间高负载的场景下不会产生Full GC导致业务中断，目前正式环境已有多个QPS:5000+的业务场景使用了CDAS，表现稳定。\n\n**（2）高度可扩展**\n\n我们建议业务在申请分片表时预估未来3~5年的数据总量来设定分片总数，前期可申请1~2个数据库，数据量变大后扩容。理论上可扩展性和分片总数相关，最大可扩展分片数相等的数据库实例，最大可扩展到250TB，同时Proxy集群也支持在线扩容。\n\n**（3）平台化运维**\n\n由云平台统一运维、部署，MySQL和Proxy都支持自动化流程申请和变更，自动接入元数据管理系统。\n\n**（4）兼容MySQL语法**\n\n支持绝大多数MySQL语法。\n\n- 路由至单数据分片100%兼容\n- 支持分页、去重、排序、分组、聚合、关联等常见查询场景\n\n**（5）功能丰富**\n\n- 多种分片算法\n- 自增分布式ID\n- 读写分离，可设置主从同步容忍阈值\n- 影子库\n- HTAP\n- 完善的监控信息：审计日志、慢日志、鉴权 \n\n## **3.核心设计** \n\nCDAS的核心目标是解决用户海量数据存储、访问问题，聚焦到分片场景中，主要体现在解析、路由、重写、聚合等逻辑 。\n\n**3.1 内核架构**\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=Y2E0NjkzNmEyZDdmMDI1ODhjZmIxMzc1NWEyMzMwZTFfbm00REp3aU9kbFhXRHNCVVdtUTlmcm1ETHlkVVhQcmNfVG9rZW46Ym94Y25HZmtiRWdWSW9MR0RjVUZqRjhDaWVnXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n站在高处看Proxy内核，主要分为连接接入模块、I/O多路复用模块、解析路由执行引擎3个大的部分，左侧可以看到线程模型也分为3块，实际涉及线程资源的还有其他子逻辑，如并行执行线程池等，接下来将围绕多个模型和执行流程了解内核架构的细节\n\n**3.2 线程模型**内核入口层基于Netty实现，从整体上看是一个典型的Netty使用场景，内核图从可以看到主要分为3个部分。 \n\n**（1）Boss Thread**负责Accept connection，也就是接受、建立连接，客户端一般都会使用连接池技术，因此建立连接的请求不会太多，单线程即可处理 \n\n**（2）I/O Threads**即Netty EventLoopGroup ，负责编解码、发起auth认证，基于Epoll事件驱动 I/O多路复用，一个线程可处理多个连接，共CPU核心数 * 2个线程。 \n\n**（3）Worker Threads**负责执行、回写数据等核心流程。Worker线程整体是同步的，其中Parser\\Router\\Rewriter\\RateLimter是纯CPU计算，Executor Engine、ResultManager底层基于JDBC规范，JDBC目前对外暴露的是一个同步的调用模型，在数据库未响应前线程状态为blocked，因此Worker线程池的配置决定了Proxy整体的并发能力。 线程资源在操作系统中是相对昂贵的资源，在CPU核心数固定的情况下，线程数量过大会耗费大量内存和导致上下文频繁切换，降低性能。我们在压测过程中额外关注了线程数对服务的影响，找到了一个合适的计算公式。\n\n```Plaintext\nMath.min(cpuNum * X, maxThreadsThreshold)\n```\n\n- X：默认75，可配置\n- maxThreadsThreshold：默认800，可配置\n\n同时我们自研了DynamicBlockingQueue，在队列积压任务到一定阈值后提前创建新线程执行任务。\n\n**3.3 连接模型**\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=NmExYWRlYzEzOTBiYmQ0N2MyZTE1YzAyYWYzOWQ3MWJfZ2oyeHM0R3FVM2lqaUEwMk9wc3E4R0p4RW53UDB4bTBfVG9rZW46Ym94Y253YVNQeWpFZUxOM2NiZGRxTEplMzRiXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n在接受客户端连接后，Proxy内部会为每个客户端连接维护1个Backend Connection的逻辑连接。Backend Connection的connections列表保存执行过程中用到的真实连接，请求执行完毕或事务结束后清空connections列表，释放物理连接。同时，Proxy内部使用连接池技术来保存物理连接，减少连接建立时间并控制总的连接数量\n\n**3.4 事务模型**接下来聊一下在事务场景中的执行流程。\n\n**3.4.1 单库事务**假设有以下事务场景：\n\n```Plaintext\nbegin;\nselect * from t_order where order_id = 1;\nupdate t_order set gmt_modify = now() where order_id = 1;\nupdate t_order_item set cnt = 2 where order_id = 1;\ncommit/rollback;\n```\n\n事务过程中一共会和数据库顺序交互5次。 语句1：begin时Proxy并不知道后面要执行什么语句，无法路由到RDS，只是在连接上记录了一个begin状态语句2：执行带分片键的select，还是以16分片为例，最终语句会路由到t_order_1这张表中，获取connection_db1，先执行begin，再执行select，并放入逻辑连接的连接列表中\n\n> connection_db1为t_order_1表所在数据库上的连接\n\n语句3、语句4：路由到t_order_1,t_order_item_1,和语句2路由到相同DB，复用connection_db1执行语句5：单纯的commit或rollback语句，取出当前逻辑连接的所有RDS连接，挨个执行commit/rollback在事务执行过程中，是否会产生分布式事务完全由用户SQL语句来控制，如果事务执行过程中不会切换数据库，则会退化成单纯的RDS transaction，保持完整的ACID特性，如果事务执行过程中出现路由到多个DB的情况，则会产生分布式事务。\n\n**3.4.2 分布式事务**目前分布式事务的解决方案主要有以下3种\n\n**（1）最终一致**\n\n业务发起的最终一致性方案，如Seata\\TCC等，业务有感，多用于跨服务调用场景，如订单和库存系统，某一环提交失败需特定逻辑来整体回滚，不适合Proxy场景\n\n**（2）强一致XA**\n\n多用于跨服务调用场景，目前存在性能不佳，协调器单点\\Recovery锁占用等问题，不适合高并发场景使用\n\n**（3）厂商提供分布式事务**\n\n分布式数据库厂商如OceanBase本身会对数据进行分片，事务执行过程中操作了多个分布式节点从而引入分布式事务。针对单个分布式库内的事务保证ACID，Proxy后端基于MySQL协议，无法在多个RDS间实现这类事务。\n\n**（4）现状**\n\nCDAS Proxy并没有提供XA来保证跨RDS实例的强一致性，也没有在内部支持最终一致性。如果触发多库事务则分开提交，有部分提交成功、部分失败的风险，因此建议业务方在设计事务时尽量不再RDS集群内部产生跨库事务。\n\n**3.5 分片模型**\n\n以4库16分片场景为例阐述CDAS分片方案和传统方案的区别\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=M2QzMmRkNGU3YWQ4ZjQxOTM0MTM5NDRmZmZiNTZkMjlfc3ZiM2Y0akdiVnBKNjZHTUgyZ1F1bG9rOTJubkR5cmxfVG9rZW46Ym94Y25TWU1xTnhaTFhvaFlhSTZFWXo1Q2xjXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n传统方案每个DB中的分片名都是从后缀0开始的，每个库中有相同的分片数量。有以下劣势：\n\n- 热点分片处理困难 \n- 无法针对分片进行迁移 \n- 分片名无意义，无法自解释 \n\n针对以上问题，我们采用了类似范围分片的思路来优化了分片名。\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ZGJkYTA5NDM3YjRmOTA3YzUzMjRjZTY2MWMyNGVjMjRfWk1uaGFVdG5mdWd3QWFEc2wxazVSeVhWSFVZak52ZXFfVG9rZW46Ym94Y253NEJRTHMxejFNVDFTNXQzM2o1SkZjXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n分片名是唯一的，位于不同DB中的分片名互不相同，这种设计方式与Redis/MongoDB中不同slot位于不同实例上很相似，有利于扩展。\n\n| 问题         | 解决方式                                                     |\n| ------------ | ------------------------------------------------------------ |\n| 热点分片     | 热点分片可以通过扩展1个新RDS来解决，如添加DB5，将某个流量特别大的热点迁移到DB5 |\n| 分片迁移     | 单个分片迁移不影响其他分片，也不会被分片算法影响             |\n| 分片名无意义 | 通过分片名唯一让每个分片能表述它在整体集合中的位置           |\n\n t_order_1为热点分片或DB1压力过大时迁移t_order_1到新的DB5中：\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=Mjg3NDVkZTE5ZDk1ZDViM2ZlZjQzN2RiOGZmYmViMGJfWk1NNEdxTXVBNmhsZGpwdEtZQ0hDZ095dTFoR3JJemhfVG9rZW46Ym94Y25rbDJLQ1dNTXB2RVdHS1Z0cTF1aWpmXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n当然这种迁移方式并不适用于所有场景，如表t_order、t_order_item的分片算法完全相同，则他们一定落到同一个DB中，如果迁移过程中只迁移其中一张表势必会导致不必要的分布式事务，因此迁移时会将绑定关系的表一并迁移。**3.6 执行流程** 以最简单的场景举例，逻辑表：t_order分16片，分片键：order_id\n\n```Plaintext\nselect * from t_order where order_id=10\n```\n\n**（1）解析**\n\nWorker线程拿到这条语句后首先要做的是解析SQL语句，基于开源产品Antlr实现语法树解析。\n\n```Plaintext\n{type：select，table：t_order，condition: order_id=10}\n```\n\n**（2）路由**\n\n在获取到分片表名和条件后，根据表名获取分片规则并计算分片\n\n```Plaintext\norder_id mod 16 => 10 mod 16 => 10\n```\n\n得到真实的table：**t_order_10**（**3）重写**重写流程会将逻辑表名替换为真实表名，并计算真实位于哪个DB中。\n\n```Plaintext\nselect * from t_order_10 where order_id=10\n```\n\n**（4）限流**\n\n限流功能可以控制某类SQL的瞬时并发度，目前仅对部分场景进行了限流，如OLAP请求。\n\n**（5）执行**\n\n语句将会发往真实的数据库中执行，目前OLTP请求使用MySQL官方Connector/J发送，OLAP请求使用ClickHouse官方SDK发送，并且AP流量不会路由+重写，因为在整体设计上ClickHouse集群中的分布式表名和逻辑表名是相同的\n\n**（6）聚合结果**\n\n聚合的主要场景是查询请求被路由到了超过一个真实表的场景，如\n\n```Plaintext\nselect * from t_order where order_id in (10,11)\n```\n\n这种情况一条SQL最终到DB层面会是两次子请求\n\n```Plaintext\nselect * from t_order_10 where order_id in (10,11)select * from t_order_11 where order_id in (10,11)\n```\n\n聚合时，通过JDBC API遍历两个子请求的ResultSet，在内存中新建一个逻辑ResultSet，当逻辑ResultSet被遍历时会触发子ResultSet被遍历，最终将逻辑ResultSet编码在MySQL协议包发往客户端。如果SQL语句中包含order by，聚合时会在内存里面进行归并。内存里面排序需要将所有行数据加载到内存中排序，内存压力会很大，因此虽然有排序能力，我们还是不建议在请求分裂多个子请求的场景中使用order by语法。\n\n**3.7 HTAP**HTAP的是指混合OLTP（Online Transactional Processing）和 OLAP（Online Analytical Processing），在数据库中同时支持在线事务请求和后台分析请求。 MySQL是一个典型的OLTP型数据库，不太适合OLAP型业务，因为分析语句一般需要执行复杂的聚合和查询大量数据，这可能会影响其在OLTP上的稳定性。同时OLAP数据库一般在存储方式上采用列存来最大化压缩效率、空间占用，而MySQL采用行存，一般情况下须先将整行记录查询出来再过滤出某些列，在执行分析请求时会有明显的IO读放大，效率不高。CDAS通过DTS（数据传输）服务将MySQL各分片数据传输并聚合到ClickHouse（分析型列存数据库），形成统一的数据视图。AP流量到达Proxy后会被转发到ClickHouse，数据响应后再以MySQL数据包的形式返回给客户端，通过这种方式，CDAS实现了基于MySQL协议的HTAP能力。\n\n## **4.部署架构**\n\n部署拓扑图如下：\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MWNiOGQ1YzAxODA2OTMxYzdhZGYwOWY1OTQzNTBhZmVfMGhLaXF6c1FsTWUyQ2hFbFhGZTYwc3Vmcnd3R0JSUHJfVG9rZW46Ym94Y25acFNIek5pMlJ6NG5BS3c1bFNSd2I4XzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n现代架构中业务方对高可用的要求越来越高，目前大多数重要的服务都会采用多机房主备或互为主备的方案来部署，一般采用双机房或双中心化方案，保证其中一个区域挂掉后仍然能对外服务。在设计之初CDAS考虑到作为L7层的数据流量入口拥有极高的高可用要求，因此允许用户申请双机房Proxy集群，保证其中1个机房挂掉后另1个机房依然可以承接流量对外服务，实现机房级的高可用。业务系统通过L4负载均衡来和Proxy建立连接，L4和Proxy紧密协调保证优雅下线以及新实例发现，并感知Proxy节点的健康状态，及时下线不可用节点。同时由L4来做同机房优先路由、流量转发，Proxy后的所有MySQL集群都采用semi-sync的高可用架构模式。这种部署架构使CDAS具备了机房级灾备的能力。\n\n## **5.性能测试**\n\n在测试过程中我们尤其关注了性能方面的损耗，并对Proxy代理、直连RDS进行了对比测试，这样就能直观的得到性能损耗程度。\n\n**5.1 压测工具**\n内部压测平台(内部基于JMeter)\n\n**5.2 压测方式**\nJava SDK + 连接池方式压测 主要分为两组\n\n- 压测Proxy，Proxy连接2个RDS集群  \n- 压测单个RDS集群，作为Proxy性能对比的基准 \n\n压测语句：参考**SysBench**的数据库压测语句，由于分片场景一般不会直接使用主键ID分片，且主键ID在InnoDB中查询效率高于二级索引，因此添加sharding_k列进行分片测试，更符合实际的应用场景。\n\n```Plaintext\nCREATE TABLE `sbctest_x` (  \n`id` bigint(11) NOT NULL,\n`sharding_k` bigint(11) NOT NULL,\n`k` int(11) DEFAULT NOT NULL,  \n`name` varchar(100) NOT NULL,  \n`ts` timestamp NULL DEFAULT NULL,  \n`dt` datetime DEFAULT NULL,  \n`c` char(100) DEFAULT NULL,  \n`f` float DEFAULT NULL,  \n`t` tinyint(4) DEFAULT NULL,  \n`s` smallint(6) DEFAULT NULL,  \nPRIMARY KEY (`id`),  \nKEY `sharding_k_idx` (`sharding_k`),\nKEY `k_idx` (`k`),\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n\n压测SQL\n#QPS 非事务的简单查询\nSELECT COUNT(1) FROM $table WHERE sharding_k=?\n\n#TPS 包含 1条查询，4条变更语句\nBEGIN;\nSELECT c FROM $table WHERE sharding_k=?\nUPDATE $table SET k=k+1 WHERE sharding_k=?\nUPDATE $table SET c=?,ts=?,dt=? WHERE sharding_k=?\nDELETE FROM $table WHERE sharding_k=?\nINSERT INTO $table (id,sharding_k,k,name,ts,dt,c,pad,f,t,s) VALUES(?,?,?,?,?,?,?,?,?,?,?)\nCOMMIT;\n```\n\n**5.3 压测报告** 以8C16G规格为例：SSD\\100分片\\单片250W行\\数据量大于InnoDB buffer pool\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MjczNGU0ZWI0OTczM2I0NzRmNDQxMGJiY2UxMTY2OTlfMFZEQWlLd3BwT1V3aDNaUEt0UjBheGZYTFJjMXVidlZfVG9rZW46Ym94Y254UzQxVW8zM0NsSlZwbzZFOFo0VmFkXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n**（1） QPS 场景**\nProxy QPS只有RDS QPS的1/2，压测过程中观察到RT约为2倍 QPS场景主要是大量非事务查询的场景，Proxy的性能损耗约1/2，QPS压测过程中我们观察到Proxy后的RDS实例CPU使用率较低，于是将Proxy的CPU规格提升1倍后，Proxy QPS立即提升1倍，接近直连RDS的性能。\n**（2） TPS 场景**\n由于TPS涉及事务提交、log等操作，I/O频繁，瓶颈也主要在I/O，即使是SSD磁盘性能也都不高。Proxy后挂载了2个RDS实例，TPS在各并发用户数中都明显高于直连单个RDS，且在并发用户数增加后可达到其2倍，性能损耗几乎可忽略。\n**（3） 结论**\n如果业务场景中非事务查询的请求占绝大多数，且RDS的CPU利用率较高，建议选择Proxy的规格时CPU要高于RDS的规格如果业务场景中事务执行请求较多，Proxy不会成为性能瓶颈，规格可和RDS保持一致\n\n## **6.案例**\n\n目前支持了多个在线业务，拥有完善的指标展示、报警机制，我们例举个业务的控制台界面及监控图。 Proxy列表：\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MWU2YjRmMWFmMDNiOGI2OTkwYjI3NWY0MmZmMzQzMTVfUnd2aGpZNVdjT3Jvck5pOEYzQXJXcDE5TFczM1pwd1BfVG9rZW46Ym94Y25uMVpueEg5emlkVUx5eWQ4d1lxeWtkXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n单个节点TPS:\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=NzYxODMxN2ZlNDRmMDNiMTI2MzliMTMyMjFlYzllMmFfb0djVHU2dThteDZueGxTQW53dlZaRG1RQ0sxOXluSjNfVG9rZW46Ym94Y25DS2JjNjFBY2cxMVRCaE5hM1BmS1ZoXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n单个节点QPS:\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ODg3ZDg2NGZkMmVkNjkzMWExYmUwMWQ1YmZkMTkzZTZfcnlwc3NKTFgyQW1MYlM5MGRZakc1dGxFdXZVZnVYVGhfVG9rZW46Ym94Y24ycEhPSzA0RGc4RThtN1ZJR1hrVU9nXzE2ODAxNTg5NDE6MTY4MDE2MjU0MV9WNA)\n\n在多个产品的使用过程中CDAS表现稳定，我们将会继续往更大流量的场景推广，服务更多的业务方。 \n\n## **7.总结与展望**\n\nCDAS为数据分片而生，构建基于MySQL分片场景的内部标准，通过MySQL协议实现跨语言可访问，统一使用方式，简化使用成本。  未来我们将持续专注于CDAS的性能提升和功能补充，主要体现在下面2个方面： \n\n1. ShardingSphere社区已计划将Proxy的数据库驱动从MySQL Connnector/J 替换为事件驱动组件vertx-mysql-client（一款Reactive MySQL Client），基于Netty实现和MySQL交互I/O多路复用、事件驱动替换后worker线程池执行语句时将从blocking等待变为事件驱动，少量的线程即可支持大量的并发，我们将持续关注社区的开发进度、积极参与。 \n\n2. CDAS和MySQL交互过程中会自动解码Rows数据包，产生大量的对象并且都是朝生夕灭的，导致GC压力增大，同时编解码也会耗费大量CPU时间。 Rows数据包在非解密等特殊场景外都可以基于Netty ByteBuf实现零拷贝发送，这是最佳的方案。我们计划在开源版本替换为vertx-mysql-client后对这部分逻辑进行优化，使绝大多数的请求的响应时间能达到接近4层负载均衡的性能。","date":"2022-03-14","author":"SphereEx","excerpt":null,"metaDescription":null,"createdAt":"2023-03-30T07:01:59.275Z","updatedAt":"2023-03-30T07:02:00.368Z","publishedAt":"2023-03-30T07:02:00.365Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":12,"attributes":{"feature":true,"title":"增强企业数据服务及管控能力｜SphereEx-Console 可视化数据管控平台正式发布","content":">**1 月 27 日，SphereEx 正式推出可视化数据服务管控平台：SphereEx-Console。**\n>\n>作为一款以管理 ShardingSphere 为核心的 SphereEx 商业数据服务集群平台，SphereEx-Console 以平台的形式简化用户配置，极大简化用户操作和部署流程，进而降低新手入门门槛和企业管控难度，进一步提升企业运维效率。\n\n随着数据体量增长以及前端业务敏捷性的要求，金融、政府、传统制造等领域均面临着提升底层数据架构适配性、面向分布式生态进行架构升级。但传统集中式数据库在海量数据承载能力方面存在先天性不足，于是近些年来分布式数据库不断涌现，很好地满足这一诉求。但我们也看到，分布式架构天然的复杂性，对使用者提出了更高的要求，如何在新架构下简化用户体验，便捷普惠地享受到新架构带来的优势，成为各家重点考虑的因素。\n\n作为近些年火热的开源项目，Apache ShardingSphere 的应用范围越来越广，越来越多的企业选择采用 “数据库 + ShardingSphere” 的方式来实现传统关系型数据库到分布式数据库的转型过渡。以 Apache ShardingSphere 为基础，在原有数据库之上构建一套数据中间平台，将分布式数据服务增强能力部署在中间层，而不需对底层数据架构做出调整，进而帮助企业以更加平滑的方式获得分布式以及云上迁移能力。\n\n因此，为进一步降低用户应用门槛，拓宽 Apache ShardingSphere 的使用范围，**SphereEx 在开源内核 Apache ShardingSphere 的基础上推出 SphereEx-Console 可视化数据管控平台。** 作为一款以管理 ShardingSphere 为核心的 SphereEx 商业数据服务集群平台，SphereEx-Console 以平台的形式简化用户配置，极大简化用户操作和部署流程，进而降低新手入门门槛和企业管控难度，进一步提升企业运维效率。\n\n## 什么是 SphereEx-Console\n\n**SphereEx 控制台主要用于对 SphereEx 数据平台的运维管控，在互联网、金融、政务行业均能够支持运维和开发人员常见的操作场景。**\n\n作为一款以管理 ShardingSphere 为核心的 SphereEx 商业数据服务集群平台，SphereEx-Console 可视化数据管控平台以简便易用的操作界面代替相对复杂代码输入方式，结合内部针对资源、实例、插件等多种功能封装，为用户带来一站式的使用体验。\n\n![1.png](https://official-media.sphere-ex.com/1_5bd2d7ea7f.png)\n\n### 产品优势\n\n**功能易用**\n\n提高了用户体验，避免了配置错误，用户不需要通过配置和命令操作 SphereEx 企业数据服务平台，易用性得到极大提升，实现了平台使用的“零”瓶颈。\n\n**功能全面**\n\n完善开源版本的管理能力，提供了对 SphereEx 企业数据服务平台从基础资源到插件能力的管控，形成了对周边插件实现统一管控的解决方案。\n\n**功能可视**\n\n原生提供可视化的监控数据面板，可以实时在线查看 SphereEx 企业数据服务平台的集群、实例、主机的监控数据。\n\n\n### 行业价值\n\n**本次 SphereEx-Console 的发布，是 SphereEx 在数据管理生态下一次关键尝试，弥补了 Apache ShardingSphere 过去在管理易用性、可视化界面等维度的不足。** SphereEx-Console 以可视的操作界面，极大降低了企业运维人员的学习成本；统一的数据管控平台，也极大提升了企业的运维管理效率。\n\n作为一家以开源项目 Apache ShardingSphere 为基本的商业化公司，SphereEx 将持续增强自身在数据治理、管控等方面的能力，配以周边完善的生态基础，为用户提供更加全面和完善的 Apache ShardingSphere 应用生态以及 SphereEx 产品矩阵能力。\n\n**[点击了解详情](https://sphere-ex.com/zh/products/console/)**","date":"2022-01-26","author":"SphereEx","excerpt":"提供企业级的平台管控能力","metaDescription":null,"createdAt":"2022-03-11T09:39:49.579Z","updatedAt":"2022-03-11T09:55:15.098Z","publishedAt":"2022-03-11T09:40:24.284Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":{"id":177,"attributes":{"name":"SphereEx-Console_画板 1.png","alternativeText":"SphereEx-Console_画板 1.png","caption":"SphereEx-Console_画板 1.png","width":3750,"height":1596,"formats":{"thumbnail":{"name":"thumbnail_SphereEx-Console_画板 1.png","hash":"thumbnail_Sphere_Ex_Console_1_3552d392b3","ext":".png","mime":"image/png","width":245,"height":104,"size":26.11,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_Sphere_Ex_Console_1_3552d392b3.png"},"large":{"name":"large_SphereEx-Console_画板 1.png","hash":"large_Sphere_Ex_Console_1_3552d392b3","ext":".png","mime":"image/png","width":1000,"height":426,"size":185.42,"path":null,"url":"https://official-media.sphere-ex.com/large_Sphere_Ex_Console_1_3552d392b3.png"},"medium":{"name":"medium_SphereEx-Console_画板 1.png","hash":"medium_Sphere_Ex_Console_1_3552d392b3","ext":".png","mime":"image/png","width":750,"height":319,"size":128.74,"path":null,"url":"https://official-media.sphere-ex.com/medium_Sphere_Ex_Console_1_3552d392b3.png"},"small":{"name":"small_SphereEx-Console_画板 1.png","hash":"small_Sphere_Ex_Console_1_3552d392b3","ext":".png","mime":"image/png","width":500,"height":213,"size":73.43,"path":null,"url":"https://official-media.sphere-ex.com/small_Sphere_Ex_Console_1_3552d392b3.png"}},"hash":"Sphere_Ex_Console_1_3552d392b3","ext":".png","mime":"image/png","size":345.65,"url":"https://official-media.sphere-ex.com/Sphere_Ex_Console_1_3552d392b3.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2022-03-11T09:27:36.623Z","updatedAt":"2022-03-11T09:27:36.623Z"}}},"metaLogo":{"data":null},"localizations":{"data":[{"id":11,"attributes":{"feature":false,"title":"The SphereEx-Console Enterprise Grade Data Governance Platform is Officially Released","content":">On January 27th, SphereEx officially launched SphereEx-Console, a data monitoring and management platform. It provides a one-stop solution incorporating Apache ShardingSphere and SphereEx as the core, a powerful user interface to manage data clusters, and a platform to simplify user configuration.\n>\n>By simplifying operation & deployment processes, reducing entry barriers and data management challenges, SphereEx-Console improves enterprise operations and maintenance efficiency. \n\n\nExpanding data volumes and increasingly demanding agility requirements from front-end businesses have created significant challenges for enterprises in the finance, government and traditional manufacturing sectors. \n\nImproving underlying data architectures' adaptability, and upgrading it to a distributed ecosystem represent some of the new requirements - however, traditional centralized databases are inherently inadequate in terms of massive data carrying capacity. \n\nAs a result, distributed databases have emerged in recent years to meet this demand. Nevertheless, the complexity of distributed architectures poses great challenges to users. Simplifying users' experience has become a key database industry driver.\n\nAs a popular open source project, Apache ShardingSphere has successfully been implemented in multiple production scenarios, with an increasing number of enterprises adopting the Database Plus approach to transition from traditional relational databases to distributed databases. \n\nBased on Apache ShardingSphere a data platform on top of the existing databases is built, enabling distributed data capability to be deployed in the middle layer without requiring adjustments to the underlying data architecture. This approach allows enterprises to smoothly transition to distributed and cloud based platforms.\n\nThe SphereEx Enterprise Data Service Platform was created by leveraging ShardingSphere's open source kernel, and enhancing it with enterprise grade features. It provides improved data service capabilities, such as data sharding, data security etc.\n\n**SphereEx-Console is a powerful dashboard applied to the management and control of the SphereEx Enterprise Data Service Platform. It provides a one-stop solution with a user-friendly experience packaging resources, instances and plugins.**\n\nIt provides a visual approach to manage data clusters with Apache ShardingSphere and SphereEx as the core, while offering a platform to simplify user configuration, and ultimately improve enterprise operations & maintenance efficiency.\n\n## What is SphereEx-Console?\n\nThe SphereEx Enterprise Data Service Platform was created by leveraging ShardnginSphere's open source kernel, and enhancing it with enterprise grade features. It provides improved data service capabilities, such as data sharding, data security etc.\n\nSphereEx-Console is a powerful dashboard applied to the management and control of the SphereEx Enterprise Data Service Platform. It provides a one-stop solution with a user-friendly experience packaging multiple functions such as resource layer, instance layer and application layer.\n\nThe SphereEx-Console is used to manage the operation and maintenance of SphereEx Enterprise Data Service Platform. It supports operation, maintenance and development scenarios in Internet, finance and government industries.\n\nAs a SphereEx business data services cluster platform with ShardingSphere at its core, the SphereEx-Console visual data control platform provides a one-stop solution with a user friendly interface instead of complex code input, packaged with various features for the resource, instance and application layers. \n\n![Diagram Console EN.png](https://official-media.sphere-ex.com/Diagram_Console_EN_5f98720b3f.png)\n\n\n### SphereEx-Console Advantages\n\n**Easy to Use**\n\nIt improves the user experience and eliminates configuration errors. Users are not required to operate the SphereEx Enterprise Data Service Platform through configuration commands, greatly improving user experience. \n\n**Comprehensive**\n\nIt provides a unified solution for the management and control of the SphereEx Enterprise Data Service Platform - from basic resources to plugin capabilities.\n\n**Visual Monitoring**\n\nWith its data monitoring & management dashboard, SphereEx Enterprise Data Service Platform's clusters, instances and hosts monitoring data are visible online and in real time.\n\n### Industry Leading\n\nThe release of SphereEx-Console is an essential milestone for SphereEx's data management ecosystem. It improves and evolves Apache ShardingSphere in areas such as management difficulty and user interface. \n\nThe user interface greatly reduces enterprise operations & maintenance teams' learning costs, while the unified data management platform greatly improves operations and maintenance efficiency.\n\nAs a business founded the open source project Apache ShardingSphere's core team, SphereEx continues to enhance data governance as well as management & control capabilities. We are committed to providing users with a comprehensive Apache ShardingSphere application ecosystem and SphereEx enterprise grade capabilities.\n\n[Click here for more details!](https://sphere-ex.com/en/products/console/)","date":"2022-01-26","author":"SphereEx","excerpt":"Enhancing Enterprise Data Governance Capabilities","metaDescription":null,"createdAt":"2022-03-11T09:38:54.812Z","updatedAt":"2022-09-27T08:42:51.504Z","publishedAt":"2022-03-11T09:59:17.395Z","locale":"en"}}]}}},{"id":1,"attributes":{"feature":true,"title":"Apache ShardingSphere 在京东白条场景的落地之旅","content":"\n>京东白条使用 Apache ShardingSphere 解决了千亿数据存储和扩容的问题，为大促活动奠定了基础。\n>\n>2014 年初，“京东白条”作为业内互联网信用支付产品，数据量爆发式的增长，每一次大促备战都是对技术人员的考验，每一次的战略转型驱动着数据架构的成长。\n>\n>— 张栋芳，京东白条研发负责人\n\n### 项目背景\n\n自 2014 年 2 月京东白条业务上线起，为满足快速发展的业务和激增的海量数据，白条的数据架构经历了数次演进。\n\n白条数据架构的演进间接地反应了互联网消费金融的飞速发展，也说明了每一种解决方案在不同背景下都有不同的保质期。\n\n### 业务挑战\n\n随着产品升级迭代，早期的解决方案演变成为了眼前的问题，通过业务框架实现的数据分片方案导致业务代码复杂度增加、维护成本不断攀升，紧耦合的弊端原形毕露，应用每次升级都需要投入较多的精力对分片做相应调整，研发人员难以专注于业务本身。\n\n技术团队经权衡后开始考虑使用成熟的分库分表组件来承担这部分工作，让业务系统升级和架构调整不再复杂。\n\n### 解决方案\n\nShardingSphere-JDBC 是 Apache ShardingSphere 的第一款产品，它定位为轻量级 Java 框架，在 Java 的 JDBC 层提供的额外服务。它使用客户端直连数据库，以 jar 包形式提供服务，无需额外部署和依赖，可理解为增强版的 JDBC 驱动，完全兼容 JDBC 和各种 ORM 框架。\n\nShardingSphere-JDBC 的以下特点能够很好地满足白条业务场景：\n\n- 产品成熟：经数年打磨产品成熟度高，且社区活跃；\n- 性能良好：微内核、轻量化的设计，性能损耗极小；\n- 改造量小：支持原生的 MySQL 协议，研发工作量小；\n- 扩展灵活：搭配使用迁移同步组件轻松实现数据扩展。\n\n经内部大量系统性验证之后，Apache ShardingSphere 成为了京东白条数据分片中间件的首选方案，2018 年底正式开始对接。\n\n![jd_zh_1.png](https://official-media.sphere-ex.com/jd_zh_1_fac28cd6ec.png)\n\n### 用户收益\n\n- 简化升级路径\n\n通过架构解耦，业务系统升级所涉及技术栈得到有效缩短，研发团队不再需要关注分表设计，精力全部聚焦于业务本身，升级路径得到大幅度优化；\n\n- 节省研发力量\n\n引入成熟的 Apache ShardingSphere 无需重新开发分表组件，在简化业务升级路径的基础上节省了大量研发力量；\n\n- 架构灵活扩展\n\n搭配使用 Scaling 同步迁移组件从容面对“618”和“11.11”等大型活动，系统灵活扩容。\n\n**SphereEx，让数据与应用连接更简单。**\n\n","date":"2021-12-01","author":"SphereEx","excerpt":"京东白条使用 Apache ShardingSphere 解决了千亿数据存储和扩容的问题，为大促活动奠定了基础。","metaDescription":"京东科技集团是京东集团旗下专注于以技术为产业服务的业务子集团，致力于为企业、金融机构、政府等各类客户提供全价值链的技术性产品与解决方案。","createdAt":"2021-12-11T23:10:03.172Z","updatedAt":"2021-12-14T07:22:15.304Z","publishedAt":"2021-12-11T23:10:07.307Z","locale":"zh","caseStudiesType":{"data":{"id":4,"attributes":{"name":"电商","createdAt":"2021-12-11T23:02:39.013Z","updatedAt":"2021-12-11T23:02:41.225Z","publishedAt":"2021-12-11T23:02:41.156Z","locale":"zh"}}},"cover":{"data":{"id":6,"attributes":{"name":"jd_cover.png","alternativeText":"jd_cover.png","caption":"jd_cover.png","width":1030,"height":720,"formats":{"thumbnail":{"name":"thumbnail_jd_cover.png","hash":"thumbnail_jd_cover_38f39465dd","ext":".png","mime":"image/png","width":223,"height":156,"size":9.38,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_jd_cover_38f39465dd.png"},"large":{"name":"large_jd_cover.png","hash":"large_jd_cover_38f39465dd","ext":".png","mime":"image/png","width":1000,"height":699,"size":74.1,"path":null,"url":"https://official-media.sphere-ex.com/large_jd_cover_38f39465dd.png"},"medium":{"name":"medium_jd_cover.png","hash":"medium_jd_cover_38f39465dd","ext":".png","mime":"image/png","width":750,"height":524,"size":47.51,"path":null,"url":"https://official-media.sphere-ex.com/medium_jd_cover_38f39465dd.png"},"small":{"name":"small_jd_cover.png","hash":"small_jd_cover_38f39465dd","ext":".png","mime":"image/png","width":500,"height":350,"size":25.77,"path":null,"url":"https://official-media.sphere-ex.com/small_jd_cover_38f39465dd.png"}},"hash":"jd_cover_38f39465dd","ext":".png","mime":"image/png","size":46.19,"url":"https://official-media.sphere-ex.com/jd_cover_38f39465dd.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2021-12-12T12:04:05.759Z","updatedAt":"2021-12-14T07:19:51.932Z"}}},"metaLogo":{"data":{"id":10,"attributes":{"name":"jd_logo.png","alternativeText":"jd_logo.png","caption":"jd_logo.png","width":300,"height":44,"formats":{"thumbnail":{"name":"thumbnail_jd_logo.png","hash":"thumbnail_jd_logo_591b2e5e14","ext":".png","mime":"image/png","width":245,"height":36,"size":6.43,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_jd_logo_591b2e5e14.png"}},"hash":"jd_logo_591b2e5e14","ext":".png","mime":"image/png","size":2.36,"url":"https://official-media.sphere-ex.com/jd_logo_591b2e5e14.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2021-12-12T12:04:06.233Z","updatedAt":"2021-12-14T07:19:09.532Z"}}},"localizations":{"data":[{"id":6,"attributes":{"feature":true,"title":"Apache ShardingSphere & JD Baitiao: Story of an Implementation Journey","content":">Apache ShardingSphere has become the best solution for JD Baitiao’s very large storage scenarios and scale-up. JD’s shopping festival weighed very much on our big data storage. I mean, like hundreds of millions of data sets.\n>\n>JD Baitiao, an internet credit products provider launched in 2014, has always been challenged by data explosion. Every shopping festival is like a technical test for us, but every time the technical staff professionals take proactive strategic initiatives in data architecture development to ensure everything goes smoothly.\n>\n>— Dongfang Zhang，JD Batiao R&D Director\n\n## The Baitiao Data Architecture Evolution\n\nLaunched in Feb. 2014, JD Baitiao’s data architecture has already evolved for several times because of rapid increase in data volume and rapid business growth.\n\n### From 2014 to 2015\n\nBaitiao first adopted the Solr + HBase solution to connect core business systems and other systems with key databases. Solr indexing is able to achieve fast search responses and HBase is used to store a massive amount of data.\n\n- A Solr cluster can help the core database reduce reads and writes stress.\n- However, Solr is not perfect: it has unsatisfying extensibility and intrusions.\n\n### From 2015 to 2016\n\nThe company chose NoSQL to store monthly data tables in Mongo DB clusters, temporarily meeting the demand for importing/exporting massive data in payment settlement scenarios.\n\n- The change made it quite efficient to query hot data. In addition, thanks to such unstructured data storage, staff can easily modify table structures.\n- Nevertheless, the plan was still problematic: bad extensibility, intrusions, and memory hog.\n\n### From 2016 to 2017\n\nGrowing business and growing data. Tens of billions of data volume imposed more performance and volume pressure on MongoDB. JD Baitiao’s Big Data Platform used DBRep and MySQL Slave to capture data changes and store the information data in its message center, to later be written to ES and HBase.\n\n- This plan was actually much better. It focused on real-time data and improved extensibility.\n- However, Baitiao faced high costs to maintain its code because of the data sharding problem in its architecture.\n\nThe architecture evolution of JD Baitiao is just an illustration of rapidly growing Internet consumer finance. All the methods Baitiao adopted in the past were not comprehensive, indicating that every solution would be outdated soon.\n\n## Time to Adopt a Decoupled Architecture\n\nTo ensure good system performance even with rapidly increasing data volumes, the technical team first chose the sharding architecture pattern that not only guaranteed excellent performance, but also kept code under control.\n\n### The data striping plan was actually based on application architecture:\n\nThe product continuously developed but the early solution had now become one of the biggest problems. The old data sharding plan makes code even more complex and increases maintenance expenditure. Developers have to spend much time adjusting sharding every time when the application upgrades, so they cannot concentrate on their own developments. In fact, tight coupling is to blame for the problem.\n\nTherefore, the team decided to use a mature sharding component that simplifies system upgrades and architecture changes. To compare Baitiao’s shard and ShardingSphere shard, their differences are shown in the following table.\n\n![jd_en_1.png](https://official-media.sphere-ex.com/jd_en_1_eb365b4a93.png)\n\nFor Baitiao, decoupling was the next task.\n\nApparently, JD Baitiao‘s data architecture will experience a brand-new journey to decoupling. The following three directions definitely fueled its transformation:\n\n**Concentration:** instead of database sharding built in its architecture, it applied a sharding component to place more energy on it own product development.\n\n**Easier Upgrade:** it used a decoupled architecture to simplify the R&D process of system upgrades.\n\n**Future-Oriented Plan:** it aims to improve system extensibility so that Baitiao will be more capable of confidently holding large online shopping festivals like the 618 Shopping Festival and Double 11 Shopping Festival (also known as Singles Day on November 11th).\n\nJD Baitiao is a huge business, so its business scenario is truly related to finance, high concurrency, and massive data volumes. Baitiao’s sharding component must have the following features:\n\n1. Be a mature product\n2. Have excellent performance\n3. Be able to handle big data\n4. Have an extensible architecture\n\n## The Apache ShardingSphere Solution\n\nThe lightweight Java framework ShardingSphere-JDBC is Apache ShardingSpheres’s first product providing services such as a Java Database Connectivity (JDBC) API. ShardingSphere JDBC uses jar archive file installation package and allows the client-side to directly connect to databases. So, it requires no extra deployment dependencies. It is like an enhanced JDBC driver fully compatible with JDBC and ORM frameworks.\n\nShardingSphere — JDBC provides the following features, making it the best solution in the Baitiao scenario.\n\nMature Product: ShardingSphere — JDBC is a mature product developed for years, and its open source community is very active.\n\nExcellent Performance: its micro-kernel and lightweight design hardly hinder performance.\n\nMinimum Changes: it supports native MySQL protocol and minimizes R&D workload.\n\nExtensibility: users can combine with the migration sync component to easily extend data.\n\n![jd_en_2.png](https://official-media.sphere-ex.com/jd_en_2_c92fb9e5ef.png)\n\nAfter being systematically tested for several times, Apache ShardingSphere became JD Baitiao’s first choice of data sharding middleware. The cooperation started at the end of 2018.\n\n## Product Adaptation\n\nTo provide Baitiao with better services and support the business, Apache ShardingSphere has made many improvements to its product features and performance during the implementation process. At the same time, the user case helps optimize the product in return.\n\n### Upgrade the SQL Engine\n\nBaitiao’s business logic is extremely complicated; its diversified scenario demands require the best SQL compatibility levels. Thus, Apache ShardingSphere restructures the SQL parser module to support more SQLs.\n\n- Single data node routing: 100% SQL compatibility\n- Multi data node routing：comprehensively support DML, DDL, DCL, TCL and partly DAL. Support features such as pagination, deduplication, ranking, grouping, aggregation and correlated query.\n\n### Distributed Key\n\nApache ShardingSphere provides built-in distributed key generators, such as UUID, and SNOWFLAKE. It also provides its distributed key generation API so that users can develop custom key generation algorithms to meet their special needs.\n\n### Shard Key Value Injection\n\nIf a SQL has no shard condition, Apache ShardingSphere uses ThreadLoad to manage shard key value, and users can program and add a shard condition to HintManager, making the condition only effective in the current thread. That’s so-called SQL zero intrusion.\n\nAdditionally, Apache ShardingSphere continues to optimize its other features in order to meet Baitiao’s need for high performance such as：\n\n- SQL parsing result cache\n- JDBC metadata cache\n- Bind table & Broadcast table\n- Automated execution engine & Stream merge\n\nJD Baitiao & ShardingSphere teams joint efforts amde all indicators of the product live up to their expectation. The resulting final performance is almost the same as that of a native JDBC.\n\n![jd_en_3.png](https://official-media.sphere-ex.com/jd_en_3_582e2afe89.png)\n\n## Cutover\n\nApache ShardingSphere uses its custom HASH strategy to shard data, effectively avoiding the hot data problem. The total of data nodes almost reaches ten thousand. The cutover process lasts for about 4 weeks.\n\n1. After DBRep reads data, Apache ShardingSphere synchronizes the data to the target database cluster.\n2. Two clusters run together. After a data migration, Baitiao uses its own tool to verify business and data.\n\nDBRep is fundamental to the product design of ShardingSphere-Scaling. The automation feature of Scaling definitely facilitates migration and scale-up.\n\n## Apache ShardingSphere Brings Many Benefits\n\n### Simplified Upgrade Path\n\nThe decoupled architecture effectively simplifies a technology stack required in a system upgrade, so the developer team no longer need to worry about table sharding. Instead, they can focus on the business itself. In brief, ShardingSphere greatly helps Baitiao optimize the upgrade path.\n\n### Save R&D Cost\n\nBaitiao directly uses the mature product Apache ShardingSphere, so they do not need to reinvent the wheel and save a lot of time and energy.\n\n### Improve Architecture Extensibility\n\nBaitiao also uses the sync migration component Scaling that makes system scale-up even more flexible. It’s the company’s secret for online shopping festival successes.\n\n## Summary\n\nThe growth of JD Baitiao’s business greatly stimulates its data architecture upgrades. This time, it chose Apache ShardingSphere to decouple its architecture and make future upgrades less complicated. Thank to this from now on, developers only have to concentrate on their own product optimization because its data architecture is extensible enough. The case is really a good example of ShardingSphere application in the consumer finance scenario.\n\nNow, there are more and more internet credit payment models. In the future, Apache ShardingSphere in collaboration with JD Technology will explore more business scenarios, and contribute finance-related technological innovations, further improving Internet finance.\n\n### About Apache ShardingSphere\n\nShardingSphere is one of the Apache Software Foundation Top-Level Open Source projects, used by over 170 enterprises worldwide, across various verticals such as finance, e-commerce, cloud services, tourism, logistics, education, and entertainment. Its GitHub community has so far cumulated over 14,000 stars.\n\nWe welcome more technical professionals to write articles and share their experiences and thoughts. If you are interested, feel free to contact us:\n\nShardingSphere Github: https://github.com/apache/shardingsphere\n\nShardingSphere Twitter: https://twitter.com/ShardingSphere\n\nShardingSphere Slack Channel: apacheshardingsphere.slack.com","date":"2021-12-01","author":"SphereEx","excerpt":"Apache ShardingSphere has become the best solution for JD Baitiao’s very large storage scenarios and scale-up. JD’s shopping festival weighed very much on our big data storage. I mean, like hundreds of millions of data sets.","metaDescription":"JD Technology Group is a business subgroup of JD Group that focuses on technology as an industry service, and is committed to providing technical products and solutions for enterprises, financial institutions, governments and other types of customers across the value chain.","createdAt":"2021-12-12T12:30:44.957Z","updatedAt":"2022-09-26T09:16:15.245Z","publishedAt":"2021-12-12T12:35:08.683Z","locale":"en"}}]}}},{"id":35,"attributes":{"feature":true,"title":"openGauss X ShardingSphere，分布式方案的另一种最佳实践","content":"Apache ShardingSphere 持续助力于 openGauss 分布式数据库能力的构建。openGauss 数据库自 2020 年 6 月开源以来，受到了业界的广泛关注，现已吸引众多伙伴、开发者参与其中，共建繁荣的数据库生态。面对如今海量数据，超高并发等诸多场景，openGauss 将目光转向于分布式解决方案，专注于解决海量数据存储、超高并发吞吐、大表瓶颈等众多难题，与 ShardingSphere 一起构建全栈开源分布式解决方案，实现 openGauss 的又一大突破。\n\n## 分布式解决方案\n![640.png](https://sphereex-media-1305704183.cos.ap-beijing.myqcloud.com/640_f6ba503847.png)\n<center>图1 分布式解决方案整体框图</center>\n\nopenGauss 融合了众多开源组件，用以构建集数据水平扩展、分布式事务及治理一体化的全栈开源分布式解决方案，整体框图如图 1 所示。\n\n- 其中 ShardingSphere-Proxy 为开源分布式解决方案，具有分库、分表、分布式事务、弹性伸缩、读写分离等众多能力；\n- HAProxy 结合 Patroni 的 REST API，可以始终识别数据库的主节点，保证高可用场景，同时可实现负载均衡；\n- 每个 Patroni 高可用节点支持一主多备，每个节点使用 Paxos 协议保证数据的一致性，各个节点可以部署在相同或不同的区域，用以保证多地多中心的数据安全。\n本分布式方案运用 ShardingSphere-Proxy 强劲的分布式能力，通过 kubernetes 管理集群，prometheus 监控集群状态，从而构建全栈开源的分布式解决方案。\n\n## 产品优势\n\n1. **极致扩展能力，灵活扩缩容**\n计算与存储能力可通过水平拆分实现线性扩展，最高可达数据 6400 分片，性能随扩展准线性增长，可有效解决单表数据量膨胀问题；结合业务流量，灵活平滑进行数据节点的扩缩容，智能读写分离，实现分布式数据库的自动负载均衡。\n\n2. **丰富企业级特性**\n支持分布式存储过程、触发器，分布式事务，全密态数据加密，WDR 诊断报告，提供丰富的企业级特性。\n\n3. **一键部署，屏蔽底层依赖**\n标准化镜像确保多环境一致性交付，容器化部署，实现物理资源池化，降低对平台的依赖性，简洁高效，实现应用秒级部署。\n\n4. **超高可用，实现异地容灾**\n强有力的集群管理、运维能力，支持同城、异地、多地多中心灵活部署，基于 Paxos 协议保证数据的安全及强一致性，提供 RPO=0 的多种容灾能力。\n\n5. **开源开放，构建全栈生态**\n开源 openGauss 单机及分布式解决方案，鼓励更多伙伴、开发者共同参与其中，共建数据库的繁荣生态，打造全栈开源生态链。\n\n## 应用场景\n\n- **金融系统**\n基于多数派共识协议 Paxos，提供分布式强一致事务能力，确保分布式环境下数据的强一致能力；多地多中心，提供 RPO=0 的能力，保障金融级可靠。\n\n- **政企办公**\n提供全密态数据加密，安全可信，支持软硬件生态，保障高可用场景，为政企办公保驾护航。\n\n- **智慧电网**\n支持同城、异地、多地多中心灵活部署，根据业务的潮汐特点灵活扩缩容，助力智慧电网。","date":"2021-09-15","author":"SphereEx","excerpt":"Apache ShardingSphere 持续助力于 openGauss 分布式数据库能力的构建","metaDescription":null,"createdAt":"2023-03-28T07:02:39.490Z","updatedAt":"2023-03-28T07:10:25.644Z","publishedAt":"2023-03-28T07:10:25.641Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":36,"attributes":{"feature":true,"title":"数据为王，携程国际火车票的 ShardingSphere 之路","content":"## 业务背景\n\n随着国际火车票业务的高速发展，订单量快速增长，单数据库瓶颈层面的问题逐渐显露，常规的数据库优化已无法达到期望的效果。同时，原先的底层数据库设计，也存在一些历史遗留问题，比如存在部分无用字段、表通过自增主键关联和各个应用直连数据库等问题。\n\n为此，经过讨论后，我们决定对订单库进行分库分表，同时对订单表进行重构，进而从根本上解决这些问题。\n\n## 业务挑战\n\n目标确定后，实践起来可不轻松，出现了很多的问题和挑战。这里列举一些典型问题，大致可以分为两大类：分库分表通用问题、具体业务关联问题。\n\n**分库分表通用问题**\n\n- 如何切分，垂直分还是水平分？分片的键，如何选取？\n- 如何根据键值路由到对应库、对应表？\n- 采用什么中间件，代理方式还是中间件的方式？\n- 跨库操作等问题，如跨库事务和跨库关联？\n- 数据扩容问题，后续如何进行扩容？\n\n**具体业务关联问题**\n\n- 各个应用直连数据如何解决？\n- 如何进行平滑过渡？\n- 历史数据如何恰当迁移？\n\n## 方案选型\n\n### **如何切分**\n\n切分方式，一般分为垂直分库、垂直分表、水平分库和水平分表四种，如何选择，一般是根据自己的业务需求决定。\n\n我们的目标是要从根本上解决数据量大、单机性能问题等问题，垂直方式并不能满足需求，所以我们选取了水平分库+水平分表的切分方式。\n\n### **分片键选取**\n\n一般是根据自己的实际业务，来选择字段来作为分片的键，同时可以结合考虑数据的热点问题 、分布问题。比如订单系统，不能根据国家字段进行分片，否则可能会出现某些国家很多的订单记录，某些国家几乎没有订单记录，进而数据分布不均。相对正确的方式，比如订单类系统，可以选择订单 ID；会员系统，可以选择会员 ID。\n\n### **如何路由**\n\n选定了分片的键之后，接下来需要探讨的问题，就是如何路由到具体的数据库和具体的表。\n\n- 映射路由\n- 分组路由\n- 哈希路由\n- 分组哈希路由\n\n在实践中，我们结合了前面的几种方式，借鉴了他们的优点不足，而采用了此种方式。因为分组方式，能很方便的进行扩容，解决了数据扩容问题；哈希方式，能解决分布相对均匀，无单点数据库热点问题。\n\n## 选择 Apache ShardingSphere 的理由\n\n在产品选型过程中，考虑了多款产品，也包括公司自己实现的产品，最终选择了 ShardingSphere，理由如下。\n\n### **技术环境**\n\n- 我们团队是 Java 体系下的，对 Java 中间件有一些偏爱\n- 更偏向于轻量级组件，可以深入研究的组件\n- 可能会需要一些个性定制化\n\n### **专业程度**\n\n- 取决于中间件由哪个团队进行维护，是否是名师打造，是否是行业标杆\n- 更新迭代频率，最好是更新相对频繁，维护较积极的\n- 流行度问题，偏向于流行度广、社区活跃的中间件\n- 性能问题，性能能满足我们的要求\n\n### **使用成本**\n\n- 学习成本、入门成本和定制改造成本\n- 弱浸入性，对业务能较少浸入\n- 现有技术栈下的迁移成本，我们当前技术栈是 SSM 体系下\n\n### **运维成本**\n\n- 高可用、高稳定性\n- 减少硬件资源，不希望再单独引入一个代理中间件，还要考虑运维成本\n- 丰富的埋点、完善的监控","date":"2021-08-24","author":"瑞华，携程高级后端开发工程师","excerpt":"国际火车票业务分库分表重构实践","metaDescription":"我们是携程火车票研发团队，负责火车票业务的开发以及创新。火车票研发在多种交通线路联程联运算法、多种交通工具一站式预定、高并发方向不断地深入探索和创新，持续优化用户体验，提高效率，致力于为全球人民买全球火车票。","createdAt":"2023-03-28T07:18:10.674Z","updatedAt":"2023-03-28T07:18:12.210Z","publishedAt":"2023-03-28T07:18:12.206Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":39,"attributes":{"feature":true,"title":"客户端分片到Proxy分片，如丝般顺滑的平稳迁移","content":"client方式的优势是实现简单，只需要通过简单的配置即可完成拆分操作。\n\n在本地通过分片进行计算，得到真实的库和表进行路由，性能相对较高。\n\n不依赖于三方，没有单点故障。 \n\n \n\nclient方式的劣势是每个项目都要去管理分片，读写分离等信息，没办法统一进行管理。\n\n当需要升级的时候只能所有项目都进行升级，没办法统一升级。\n\n最难的点在于需要推动各个业务域进行升级，升级的周期较长，对业务方的压力也比较大。\n\n相对于proxy方式，client方式连接数占用的比较多，一个数据源10个连接，部署10个实例就是100个连接。\n\n \n\nproxy方式指的是部署一个独立的服务，这个服务会实现Mysql协议，应用中只需要连接这个独立的proxy服务，把它当做一个完整的独立的数据库使用即可。\n\n分库分表等规则全部由这个proxy去管理，对应用透明。\n\n \n\n彩虹桥DAL是得物基础架构组研发的proxy方式的代理服务，有了彩虹桥就可以统一管理所有数据库，可以对数据库操作做限流保护，可以做压测时的数据自动进入影子库，可以监控慢SQL等等。\n\n \n\n**接入方案**\n\n**直接修改老数据源-改成连接彩虹桥**\n\n\n\n最简单，最快速的方式就是直接将所有老的数据源改成新的代理数据源，然后重新发布就可以了。\n\n这个方案确实快，但是不够好，不够完美，不够稳定，为啥？\n\n\n\n这其实跟改接口是一样的道理，当某天加了个新的需求，你会发现直接在老接口的基础上改一下就完成了。改完后测试发现当前没问题，但是老版本的产品还在用，然后兼容出问题了。这种场景下如果你是对接口进行升级，之前是v1，现在改成v2，老的逻辑完全不动，就完美了呀。\n\n\n\n如果我们直接改成最新的方式，那么如何平滑的发布上线呢？\n\n \n\n全部发布，万一彩虹桥那边配置有问题，或者有其他的问题导致不可用，你这不就尴尬了么，当然也可以回滚历史版本。\n\n \n\n另一种方式就是灰度发布一台机器，进行测试，没问题后全部发布，有问题就不发，其实也不错。就怕过了一段时间，万一彩虹桥有Bug, 我们还是得回滚到接入彩虹桥之前的版本。\n\n \n\n**新增一套数据源 - 支持开关切换到新数据源，下线老数据源**\n\n \n\n新增数据源，支持开关切换就跟升级API版本是一样的逻辑。保持老逻辑不变，上线时还是走老的数据源，然后通过开关动态切换到新的数据源，完成上线动作。\n\n \n\n当然这边也会出现上面提到的问题，比如彩虹桥不可用之类的情况，也可以通过灰度配置的方式来测试。将开关配置灰度到某个节点上，只切这个节点的数据源，进行测试，没问题后可以扩大范围。有问题开关一关就还原到之前的逻辑了，无影响。\n\n \n\n如果用了一段时间，彩虹桥有Bug的话同样改下开关的配置即可回滚，也不用重新发布项目。\n\n \n\n此方案不足的点就是会占用一分部数据库的连接，因为老的数据源还存在。此时需要改代码去掉老的数据源，就能释放了。如果后面还要去改代码，那还能叫完美的方案么？\n\n \n\n这点我也考虑到了，还是通过配置开关来关闭老的数据源，但是这个操作得重启服务，重启后就只有一套数据源了。（**请确保以后只用彩虹桥的方式才用此开关**）\n\n \n\n**实现步骤**\n\n1. 配置2套新数据源，连接彩虹桥的地址，订单的库分为老订单库和新的分库分表库。\n2. 新增2个动态数据源，用于开关动态切换，使用AbstractRoutingDataSource管理。\n\n![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=NTk4N2JjMTE5ZWM2ZDYzNzE3YTc4ZjJiNmNkNjU4MDNfWkxpTWY5N1djdFUwWVRaQ1ExMmEwbDZnUllkd01SeWtfVG9rZW46Ym94Y25IQ25iS3Z3SEo4QVN0TjlrV0JlNFBkXzE2ODAwNzQxMDA6MTY4MDA3NzcwMF9WNA)\n\n3. 老数据源通过@ConditionalOnProperty来控制是否加载，默认加载，用于后期下线老数据源。\n\n4. 读写分离兼容client和proxy方式。\n\n5. 分片算法重写，之前用的Sharding-Jdbc3.X版本，新的彩虹桥基于5.X版本深度定制开发，在自定义算法这块有变化，目前彩虹桥的分片算法全部在彩虹桥的扩展包中，不在订单里面。\n\n \n\n**注意事项**\n\n**select last_insert_id()不支持** \n\n \n\n在insert中通过select last_insert_id()实时返回当前插入的自增ID场景需要修改，目前订单中就一个地方用到了，而且上层其实没消费这个ID, 如果后面有其他场景需要获取刚插入的ID可以手动提前获取分布式ID，然后再用这个ID存到表中。\n\n \n\n**强制走从节点查询**\n\n \n\n老的读写分离用的Sharding-Jdbc做的，正常情况下默认查询走从节点，其他走主节点。而现在订单里面默认都是走主节点，如果需要强制走从节点需要使用HintManagerHolder.clear() 清空Hint信息来实现。\n\n \n\n猜测最开始没有从节点，业务都走主节点。后面加了从节点，如果用默认的方式查询都会走从节点，但是在某些业务场景中一致性比较高，必须走主节点查询，相对于标记出哪些查询走从节点来说，默认全部走主节点更稳。然后根据业务场景再标记是否要走从节点，这样几乎不会影响老的逻辑。\n\n \n\n接入彩虹桥后，默认是彩虹桥和Sharding-Jdbc两套数据源共存的，所以在HintManagerHolder.clear() 这块也需要做兼容，同时支持彩虹桥和Sharding-Jdbc方式的强制走从节点。\n\n \n\n**第一步：兼容老代码**\n\n \n\n将所有使用HintManagerHolder.clear()改成SqlHintUtils.clear()。\n\n```java\npublic class SqlHintUtils {\n    /*      清空Hint信息     */    \n    public static void clear() {        // ShardingJdbc        \n    \tHintManagerHolder.clear();        // 彩虹桥DAL        \n    \tBifrostContext.clearNotAutoClearPrimary();    }\n}\n```\n\n \n\n**第二步：新增注解强制走从节点，比代码更优雅** \n\n \n\n以后有新的查询需要走从节点就在Dao方法上增加@ForceSlave注解即可，此注解只能作用于Dao方法上，加在其它层无效。如果Dao方法上加了注解，那么方法内所有的查询操作都将走从节点。\n\n \n\n老的clear相关的代码其实可以用注解代替，但为了保险起见还是不改变原有的方式，新的可以用注解的方式。","date":"2021-08-13","author":"SphereEx","excerpt":"随着订单数量的增多，以及大促时需要扛住比平时多N倍的流量，单库单表的瓶颈日益显现。\n需要对数据库进行水平拆分，目前订单使用的是客户端分片的方式进行拆分，采用Sharding-Jdbc框架实现。","metaDescription":null,"createdAt":"2023-03-29T07:22:28.685Z","updatedAt":"2023-03-29T07:22:29.697Z","publishedAt":"2023-03-29T07:22:29.693Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":44,"attributes":{"feature":true,"title":"vivo 全球商城：订单中心架构设计与实践","content":"## 一、背景\n\n   随着用户量级的快速增长，vivo 官方商城 v1.0 的单体架构逐渐暴露出弊端：模块愈发臃肿、开发效率低下、性能出现瓶颈、系统维护困难。\n\n   从2017年开始启动的 v2.0 架构升级，基于业务模块进行垂直的系统物理拆分，拆分出来业务线各司其职，提供服务化的能力，共同支撑主站业务。\n\n   订单模块是电商系统的交易核心，不断累积的数据即将达到单表存储瓶颈，系统难以支撑新品发布和大促活动期间的流量，服务化改造势在必行。\n\n   本文将介绍 vivo 商城 订单系统建设的过程中遇到的问题和解决方案，分享架构设计经验。\n\n## 二、系统架构\n\n   将订单模块从商城拆分出来，独立为订单系统，使用独立的数据库，为商城相关系统提供订单、支付、物流、售后等标准化服务。\n\n   系统架构如下图所示：\n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MjJiMzQwYmM3MTNhNTAxMjFjMmZmMDEwOGY4ZjExMzhfVTYyOUZLTDNOckRKeDJEUDBheGV2dWZXZG9qbGxOZWZfVG9rZW46Ym94Y251akVGY2dJQTV6YjdmVm1kbjVLcWxkXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n## 三、技术挑战\n\n### 3.1 数据量和高并发问题\n\n   首先面对的挑战来自存储系统：\n\n   **数据量问题**\n   - 随着历史订单不断累积，MySQL中订单表数据量已达千万级。\n   - 我们知道InnoDB存储引擎的存储结构是B+树，查找时间复杂度是O(log n)，因此当数据总量n变大时，检索速度必然会变慢， 不论如何加索引或者优化都无法解决，只能想办法减小单表数据量。\n   - 数据量大的解决方案有：**数据归档、分表**\n\n   **高并发问题**\n   - 商城业务处于高速发展期，下单量屡创新高，业务复杂度也在提升，应用程序对MySQL的访问量越来越高。\n   - 单机MySQL的处理能力是有限的，当压力过大时，所有请求的访问速度都会下降，甚至有可能使数据库宕机。\n   - 并发量高的解决方案有：**使用缓存、读写分离、分库**\n\n   下面对这些方案进行简单描述：\n\n   **数据归档** \n   - 订单数据具备时间属性，存在热尾效应，大部分情况下检索的都是最近的订单，而订单表里却存储了大量使用频率较低的老数据。 \n   - 那么就可以将新老数据分开存储，将历史订单移入另一张表中，并对代码中的查询模块做一些相应改动，便能有效解决数据量大的问题。\n\n   **使用缓存**\n   - 使用Redis作为MySQL的前置缓存，可以挡住大部分的查询请求，并降低响应时延。\n   - 缓存对商品系统这类与用户关系不大的系统效果特别好，但对订单系统而言，每个用户的订单数据都不一样，缓存命中率不算高，效果不是太好。\n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=NTAyNDg5ODJjOTJlMGQwYjExMzk4ZDA3NjY4ZGIyMTZfc25Wd0xRTlVHREJFeFBNZUphMHE3NzJnb1JDS2tweDVfVG9rZW46Ym94Y242eDlCb1l6UEpmWHdsaE44NWtWcHJMXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n   **读写分离**\n   - 主库负责执行数据更新请求，然后将数据变更实时同步到所有从库，用多个从库来分担查询请求。 \n   - 但订单数据的更新操作较多，下单高峰时主库的压力依然没有得到解决。且存在主从同步延迟，正常情况下延迟非常小，不超过1ms，但也会导致在某一个时刻的主从数据不一致。\n   - 那就需要对所有受影响的业务场景进行兼容处理，可能会做一些妥协，比如下单成功后先跳转到一个下单成功页，用户手动点击查看订单后才能看到这笔订单。\n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MDE2MzcxMDM5YTg0NDc2MGFhZDFmZWZiMTIxMGI0NWRfdE9leENINGxLU01XSjU3Z29UN1Vselk5dmZ2ZFNaaFZfVG9rZW46Ym94Y25IM3lpSWRvRkFkb0puWUdhOEVDc3FkXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n   **分库**\n   - 分库又包含垂直分库和水平分库。 \n   - **① 水平分库：**把同一个表的数据按一定规则拆到不同的数据库中，每个库可以放在不同的服务器上。\n   - **② 垂直分库：**按照业务将表进行分类，分布到不同的数据库上面，每个库可以放在不同的服务器上，它的核心理念是专库专用。\n\n   **分表**\n   - 分表又包含垂直分表和水平分表。\n\n   - **① 水平分表：**在同一个数据库内，把一个表的数据按一定规则拆到多个表中。\n\n   - **② 垂直分表：**将一个表按照字段分成多表，每个表存储其中一部分字段。\n\n     \n\n   我们综合考虑了改造成本、效果和对现有业务的影响，决定直接使用最后一招：**分库分表**\n\n​    \n\n### 3.2 分库分表技术选型\n\n   分库分表的技术选型主要从这几个方向考虑：\n\n   1. 客户端sdk开源方案\n   2. 中间件proxy开源方案\n   3. 公司中间件团队提供的自研框架\n   4. 自己动手造轮子\n\n​    \n\n   参考之前项目经验，并与公司中间件团队沟通后，采用了开源的 **Sharding-JDBC** 方案。现已更名为Sharding-Sphere。\n\n   - Github：https://github.com/sharding-sphere/\n   - 文档：官方文档比较粗糙，但是网上资料、源码解析、demo比较丰富\n   - 社区：活跃\n   - 特点：jar包方式提供，属于client端分片，支持xa事务\n\n​    \n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MmJhYWUwMmUzODY0YjkyZDc0MmQ0MjY4MzY4MGRjNzFfSmhFRFdMSG9CbXc0cFhPcW9IV2RTNWRsMkk0YnBSWVVfVG9rZW46Ym94Y25FaWpyOFZabXh0MWJSb3IyZnhUTmRmXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n​    \n\n#### **3.2.1 分库分表策略**\n\n   结合业务特性，选取用户标识作为分片键，通过计算用户标识的哈希值再取模来得到用户订单数据的库表编号.\n\n    假设共有n个库，每个库有m张表，\n\n   则库表编号的计算方式为：\n\n   \\- 库序号：Hash(userId) / m % n \n\n   \\- 表序号：Hash(userId) % m\n\n​    \n\n   路由过程如下图所示：\n\n​    \n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=YjdmMzI1NjViOGYxOTk5MTVjYTk1YjgzMmFjZGM3NTVfZDZLZ0VHZm1wUDZxb1VoTHM2U3Vsc0hvOFM1eEhvQTNfVG9rZW46Ym94Y25DYTZXNEdDbjZycDBiWnRqMGVPSFNlXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n​    \n\n​    \n\n#### **3.2.2 分库分表的局限性和应对方案**\n\n​    \n\n   分库分表解决了数据量和并发问题，但它会极大限制数据库的查询能力，有一些之前很简单的关联查询，在分库分表之后可能就没法实现了，那就需要单独对这些Sharding-JDBC不支持的SQL进行改写。\n\n​    \n\n   除此之外，还遇到了这些挑战：\n\n​    \n\n   **（1）全局唯一ID设计**\n\n​    \n\n   分库分表后，数据库自增主键不再全局唯一，不能作为订单号来使用，但很多内部系统间的交互接口只有订单号，没有用户标识这个分片键，如何用订单号来找到对应的库表呢？\n\n​    \n\n   原来，我们在生成订单号时，就将库表编号隐含在其中了。这样就能在没有用户标识的场景下，从订单号中获取库表编号。\n\n​    \n\n   **（2）历史订单号没有隐含库表信息**\n\n​    \n\n   用一张表单独存储历史订单号和用户标识的映射关系，随着时间推移，这些订单逐渐不在系统间交互，就慢慢不再被用到。\n\n​    \n\n   **（3）管理后台需要根据各种筛选条件，分页查询所有满足条件的订单**\n\n​    \n\n   将订单数据冗余存储在搜索引擎Elasticsearch中，仅用于后台查询。\n\n​    \n\n### 3.3 怎么做 MySQL 到 ES 的数据同步\n\n​    \n\n   上面说到为了便于管理后台的查询，我们将订单数据冗余存储在Elasticsearch中，那么，如何在MySQL的订单数据变更后，同步到ES中呢？\n\n​    \n\n   这里要考虑的是数据同步的时效性和一致性、对业务代码侵入小、不影响服务本身的性能等。\n\n​    \n\n   **MQ 方案**\n   - ES更新服务作为消费者，接收订单变更MQ消息后对ES进行更新\n\n​    \n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ODNkZWE0Mzc3Mzc1ZWM1MjQzNmFjY2IzZTkzNmMxYjBfVWhqcjRuY2JFUWNrcVlpMkdMQmlWYU9wYUdzUVMwcmhfVG9rZW46Ym94Y25vU213TGFiVDRobEZQbnRaelp1VkxkXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n​    \n\n   **Binlog方案**\n   - ES更新服务借助canal等开源项目，把自己伪装成MySQL的从节点，接收Binlog并解析得到实时的数据变更信息，然后根据这个变更信息去更新ES。\n\n​    \n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=OGZjMzZlZTkxYzk3NmRlYTE2MzJmYmM2ZTAyMmEyZTdfNkxDSjlOeDFob2RYTklJRGp1cmhlbnhhdEZKb21lTkRfVG9rZW46Ym94Y245Rm1xbVVtTGRjWWN0UElXSGcwRGpkXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n​    \n\n   其中BinLog方案比较通用，但实现起来也较为复杂，我们最终选用的是MQ方案。\n\n   因为ES数据只在管理后台使用，对数据可靠性和同步实时性的要求不是特别高。\n\n   考虑到宕机和消息丢失等极端情况，在后台增加了按某些条件手动同步ES数据的功能来进行补偿。\n\n​    \n\n### 3.4 如何安全地更换数据库\n\n   如何将数据从原来的单实例数据库迁移到新的数据库集群，也是一大技术挑战\n\n   不但要确保数据的正确性，还要保证每执行一个步骤后，一旦出现问题，能快速地回滚到上一个步骤。\n\n   我们考虑了停机迁移和不停机迁移的两种方案：\n\n​    \n\n   **（1）不停机迁移方案：**\n\n​    \n\n   - 把旧库的数据复制到新库中，上线一个同步程序，使用 Binlog等方案实时同步旧库数据到新库。\n   - 上线双写订单新旧库服务，只读写旧库。\n   - 开启双写，同时停止同步程序，开启对比补偿程序，确保新库数据和旧库一致。\n   - 逐步将读请求切到新库上。\n   - 读写都切换到新库上，对比补偿程序确保旧库数据和新库一致。\n   - 下线旧库，下线订单双写功能，下线同步程序和对比补偿程序。\n\n​    \n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MjBjZWE5ZTRjMzQ5YTIxNGZkYjg2ODA1NzRiMGMxZDdfUUl5MU0wRWtTdUxrb3JoTnJLZzNvSDM4dXJSOEJrbHZfVG9rZW46Ym94Y25sTGFNMllSb1FmM2dUSTdwbFJLVHBnXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n​    \n\n   **（2）停机迁移方案：**\n\n​    \n\n   - 上线新订单系统，执行迁移程序将两个月之前的订单同步到新库，并对数据进行稽核。\n   - 将商城V1应用停机，确保旧库数据不再变化。\n   - 执行迁移程序，将第一步未迁移的订单同步到新库并进行稽核。\n   - 上线商城V2应用，开始测试验证，如果失败则回退到商城V1应用（新订单系统有双写旧库的开关）。\n\n​    \n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=YjhmZTZiM2RlY2ZjZTIwNTFjODY5NjM0ODcxOTA0MjdfTjZKSWJZWHoyV2M5bk9iUk1DQ2QwT08yWFlaZUhXblRfVG9rZW46Ym94Y242Tmt5NzlDbUo0SG1RZjJpdnFGcWpiXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n​    \n\n   考虑到不停机方案的改造成本较高，而夜间停机方案的业务损失并不大，最终选用的是停机迁移方案。\n\n​    \n\n### 3.5 分布式事务问题\n\n   电商的交易流程中，分布式事务是一个经典问题，比如：\n\n   - 用户支付成功后，需要通知发货系统给用户发货。\n\n   - 用户确认收货后，需要通知积分系统给用户发放购物奖励的积分。\n\n     \n\n   我们是如何保证微服务架构下数据的一致性呢？\n\n   不同业务场景对数据一致性的要求不同，业界的主流方案中，用于解决强一致性的有两阶段提交(2PC)、三阶段提交(3PC)，解决最终一致性的有TCC、本地消息、事务消息和最大努力通知等。\n\n   这里不对上述方案进行详细的描述，介绍一下我们正在使用的本地消息表方案：在本地事务中将要执行的异步操作记录在消息表中，如果执行失败，可以通过定时任务来补偿。\n\n   下图以订单完成后通知积分系统赠送积分为例。\n\n​    \n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=MjVkN2Q1NjMyYmRmZmJhODJhNmVmZmVhMGIyNTNkMmRfUDNDZmpnNmFIRllVMzBsODJkd3ZiUmtwV2RKRmxxQTZfVG9rZW46Ym94Y25yNTRtbFZOZGlJcHUwYlNueUdKTHpZXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n​    \n\n### 3.6 系统安全和稳定性\n\n​    \n\n   **网络隔离**\n   - 只有极少数第三方接口可通过外网访问，且都会验证签名，内部系统交互使用内网域名和RPC接口。\n      **并发锁**\n   - 任何订单更新操作之前，会通过数据库行级锁加以限制，防止出现并发更新。\n      **幂等性**\n   - 所有接口均具备幂等性，不用担心对方网络超时重试所造成的影响。\n      **熔断**\n   - 使用Hystrix组件，对外部系统的实时调用添加熔断保护，防止某个系统故障的影响扩大到整个分布式系统中。\n      **监控和告警**\n   - 通过配置日志平台的错误日志报警、调用链的服务分析告警，再加上公司各中间件和基础组件的监控告警功能，让我们能够能够第一时间发现系统异常。\n\n​    \n\n### 3.7  踩过的坑\n\n   **采用MQ消费的方式同步数据库的订单相关数据到ES中，遇到的写入数据不是订单最新数据问题**\n\n​    \n\n   下图左边是原方案：\n\n   在消费订单数据同步的MQ时，如果线程A在先执行，查出数据，这时候订单数据被更新了，线程B开始执行同步操作，查出订单数据后先于线程A一步写入ES中，线程A执行写入时就会将线程B写入的数据覆盖，导致ES中的订单数据不是最新的。\n\n   解决方案是在查询订单数据时加行锁，整个业务执行在事务中，执行完成后再执行下一个线程。\n\n​    \n\n   ![img](https://u01f1kqxrl.feishu.cn/space/api/box/stream/download/asynccode/?code=ZWUxYjQ1MzZjMjg5ZWE3YWY2YmQwYzBmM2NkNjY1NmRfOEpPSXNwUGhuSWpQNDZLdGlGU0pWMlR1dXhVZXhZaVBfVG9rZW46Ym94Y25MelhveURtTkxsTHNlMWhjbXJqakhnXzE2ODAxNTk4MzI6MTY4MDE2MzQzMl9WNA)\n\n   **sharding-jdbc 分组后排序分页查询出所有数据问题** \n\n​    \n\n   示例：select a  from  temp group by a，b order by a  desc limit 1,10。 \n\n   执行是Sharding-jdbc里group by 和 order by 字段和顺序不一致是将10置为Integer.MAX_VALUE, 导致分页查询失效。\n\n   ```java\n   io.shardingsphere.core.routing.router.sharding.ParsingSQLRouter#processLimit\n   \n   private void processLimit(final List<Object> parameters, final SelectStatement selectStatement, final boolean isSingleRouting) {\n        boolean isNeedFetchAll = (!selectStatement.getGroupByItems().isEmpty() || !selectStatement.getAggregationSelectItems().isEmpty()) && !selectStatement.isSameGroupByAndOrderByItems();\n       selectStatement.getLimit().processParameters(parameters, isNeedFetchAll, databaseType, isSingleRouting);\n   }\n   \n   io.shardingsphere.core.parsing.parser.context.limit.Limit#processParameters\n   \n   /**\n   * Fill parameters for rewrite limit.\n   *\n   * @param parameters parameters\n   * @param isFetchAll is fetch all data or not\n   * @param databaseType database type\n   * @param isSingleRouting is single routing or not\n   */\n   public void processParameters(final List<Object> parameters, final boolean isFetchAll, final DatabaseType databaseType, final boolean isSingleRouting) {\n       fill(parameters);\n       rewrite(parameters, isFetchAll, databaseType, isSingleRouting);\n   }\n   \n   \n   private void rewrite(final List<Object> parameters, final boolean isFetchAll, final DatabaseType databaseType, final boolean isSingleRouting) {\n       int rewriteOffset = 0;\n       int rewriteRowCount;\n       if (isFetchAll) {\n           rewriteRowCount = Integer.MAX_VALUE;\n       } else if (isNeedRewriteRowCount(databaseType) && !isSingleRouting) {\n            rewriteRowCount = null == rowCount ? -1 : getOffsetValue() + rowCount.getValue();\n       } else {\n          rewriteRowCount = rowCount.getValue();\n       }\n       if (null != offset && offset.getIndex() > -1 && !isSingleRouting) {\n          parameters.set(offset.getIndex(), rewriteOffset);\n        }\n        if (null != rowCount && rowCount.getIndex() > -1) {\n           parameters.set(rowCount.getIndex(), rewriteRowCount);\n         }\n   }\n   \n   ```\n\n​    \n\n   正确的写法应该是  select a  from  temp group by a desc ，b limit 1,10 ；  使用的版本是sharing-jdbc的3.1.1。\n\n   ES分页查询如果排序字段存在重复的值，最好加一个唯一的字段作为第二排序条件，避免分页查询时漏掉数据、查出重复数据，比如用的是订单创建时间作为唯一排序条件，同一时间如果存在很多数据，就会导致查询的订单存在遗漏或重复，需要增加一个唯一值作为第二排序条件或者直接使用唯一值作为排序条件。\n\n\n\n## 四、成果\n\n   - 一次性上线成功，稳定运行了一年多\n\n   - 核心服务性能提升十倍以上\n\n   - 系统解耦，迭代效率大幅提升\n\n   - 能够支撑商城至少五年的高速发展\n\n     \n\n## 五、结语\n\n   我们在系统设计时并没有一味追求前沿技术和思想，面对问题时也不是直接采用主流电商的解决方案，而是根据业务实际状况来选取最合适的办法。\n\n   个人觉得，一个好的系统不是在一开始就被大牛设计出来的，一定是随着业务的发展和演进逐渐被迭代出来的，持续预判业务发展方向，提前制定架构演进方案，简单来说就是：走到业务的前面去！","date":"2020-12-23","author":"SphereEx","excerpt":null,"metaDescription":null,"createdAt":"2023-03-30T07:13:27.915Z","updatedAt":"2023-03-30T07:13:28.692Z","publishedAt":"2023-03-30T07:13:28.687Z","locale":"zh","caseStudiesType":{"data":null},"cover":{"data":null},"metaLogo":{"data":null},"localizations":{"data":[]}}}],"en":[{"id":15,"attributes":{"feature":true,"title":"Apache ShardingSphere & Energy Monster","content":"[Energy Monster](https://ir.enmonster.com/)’s application of [ShardingSphere-JDBC](https://shardingsphere.apache.org/document/current/en/overview/#shardingsphere-jdbc)\n\n[Energy Monster](https://ir.enmonster.com/) is a consumer tech company with the mission to energize everyday life. The company is the largest provider of mobile device charging services in Asia.\n\nAs the company’s business concurrency volume is getting larger, the amount of data generated (users, orders, activities, etc.) increases each day. The traditional relational database has proven to be inadequate in supporting millions or tens of millions of data volumes in a single database or table.\n\nPerformance has been unable to meet the benchmark requirements of business development. Under these circumstances, data sharding is an effective way to solve the problem.\n\n## Technology selection\nUnder the [Database Plus](https://shardingsphere.apache.org/) concept, ShardingSphere is designed to build an ecosystem on top of heterogeneous databases. The goal is to provide globally scalable and enhanced computing capabilities while maximizing the original database computing capabilities.\n\nThe interaction between applications and databases becomes oriented towards the Database Plus standard, therefore minimizing the impact of database fragmentation on upper-layer services.\n\nWithin the ShardingSphere ecosystem, [ShardingSphere-JDBC](https://shardingsphere.apache.org/document/current/en/overview/#shardingsphere-jdbc) is positioned as a lightweight Java framework, providing additional services in Java’s JDBC layer.\n\nIt uses the client to directly connect to the database and provide services in the form of a jar package, without additional deployment and dependence. It can be understood as an enhanced version of the JDBC driver, which is fully compatible with JDBC and various ORM frameworks.\n\nShardingSphere-JDBC enables developers to focus only on the work outside the data layer by coordinating the data read and write under the data sharding, instead of using business code to manually select databases and tables.\n\n## Business case\nUCS is Energy Monster’s user-centric service providing basic functionality for users on the Server side. In 2018, it was stripped from [PHP](https://www.php.net/) Server and moved to the Java technology stack to implement microservitization.\n\nIt involves the design of new databases and tables and data cleaning and migration. The whole switchover process was expected to ensure the following functions:\n\n- Stability: smooth release in a short time without halting.\n\n- Accuracy: ensure accurate cleaning of tens of millions of data volumes.\n\n- Scalability: solve the performance problems caused by increasing data volume and ensure scalability.\n\n> Solutions to data cleansing and migration\n\n- Initial data synchronization.\n- The application’s server cuts off the entry (users).\n- Data synchronization (updates and new users since the last time point).\n- Data cleaning.\n- User center release.\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/2t8jnqgh5pfimhxa07id.png)\n\n\n> Data sharding strategy\n\nThe database adopts a database shards design, divided into 16 databases. The default shard key is `user_id` and the default sharding strategy `user_id` is mod 16, such as `${user_id % 16}` for the user table. For SQL that does not carry shard keys, broadcast routing is used.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/t6nihbk4du8d9h1sd5sf.png)\n\n\n`user_id` is used as the shard key because `user_id` can cover most business scenarios, and other fields possibly can be empty. In the local test, the query of shard key strategy (openId,mobile) took 50ms to 200ms.\n\n> Using the sharding algorithm\n\nThere are currently three sharding algorithms available.\n\n- Standard sharding algorithm. It corresponds to `StandardShardingAlgorithm`, used for scenarios that use a single key as the shard key, such as =, IN, BETWEEN AND, >, <, > =, < =.\n- Complex sharding algorithm. It corresponds to `ComplexKeysShardingAlgorithm`, used for scenarios that use multi-key as the shard key. The logic with multiple shard keys is complex and requires developers to handle it by themselves.\n- Hint sharding algorithm. It corresponds to `HintShardingAlgorithm`, used for scenarios where the Hint row is used for sharding.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/5uv7u0nrglequ83gn4ky.png)\n\n\n## Upgrading ShardingSphere-JDBC\n\nShardingSphere-JDBC is used in multiple business scenarios, such as order, inventory, and finance. By 2021, the R&D groups or teams were using different versions of ShardingSphere-JDBC, ranging from 1.X to 4.X, which is difficult to achieve unified maintenance in the later stage.\n\nAdditionally, there are some potential bugs and missing functions in the earlier version. Based on requirements for unified management and availability, we implemented a project to unify the ShardingSphere-JDBC’s versions used by the company and upgrade them to a 4.1.1 stable version in April 2021.\n\n**The following problems were encountered during the upgrade:**\n\n**1. It takes a long time to start the service after the upgrade.**\n\nShardingSphere-JDBC checks the metadata consistency of sub-tables when the service is started. The configuration item `max.connections.size.per.quer` (maximum number of connections that can be opened per query) is 1 by default. With a large number of tables, the loading process would be slow. You need to refer to the connection pool configuration to improve the loading speed.\n\n**2. There is no response when there is no shard key in the sub-table query.**\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/641dysf5zoizgr8bmjuq.png)\n\n\nLogical SQL query does not specify shard keys and it queries all the tables according to the whole database tables router in broadcasting routing.\n\nThe configuration items have 108 pieces of real tables in a database. According to the configuration of `maxConnectionsizeperquery=50`, ShardingSphere-JDBC uses the connection limit mode, divides the query requests into three groups, and merges the results with in-memory. As a result, 36 database connections are required for one query. But the `maxActive` configured by the [druid](https://druid.apache.org/) thread pool is set to 20, resulting in a deadlock.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/ydv03pncno3wsd23y7r1.png)\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/61blt38kcoukha8aguob.png)\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/2c2w85mdryc53gh4ope5.png)\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/568lubr5zg52ropjw374.png)\n\n\n\n**Solutions:**\n\n- Combine `check.table.metadata.enabled=true`（check the metadata consistency in sub-tables when started）and properly configure - `maxConnectionSizePerQuery`（maximum number of connections that can be opened by each query).\n- `maxConnectionSizePerQuery` should be less than the maximum number of active threads configured by the druid thread pool.\n\n**3. After upgrading from 1.X, an error message “Cannot update Sharding key” is displayed in SQL execution, and the actual shard key value is not updated.**\n\nTo avoid data query failure caused by changing the shard key value, shard key detection is added to the `SQL update` in the 4.X version. The error can be rectified in the following ways:\n\n- remove the shard key when updating.\n- the shard key is added to the `where` statement synchronously.\n\n**4. A start failure is caused when using `druid-spring-boot-starter`, which is incompatible with `Sharding-datasource`.**\n\nThe druid data connection pool starter will load and create a default data source. This will cause conflicts when ShardingSphere-JDBC creates data sources.\n\n**5. `inline strategy` reports an error in range query.**\n\nThe `inline strategy` doesn't support range query by default and the `standard strategy` is advised. Add the following configuration if the `inline strategy` is needed for the range query.\n\n`spring.shardingsphere.props.allow.range.query.with.inline.sharding: true`\n\n**Note:** Here all the `inline strategy` range queries will query each sub-table in broadcasting.\n\n**6. The “Cannot find owner from table” error is reported.**\n\nSQL (simplified):\n\n`select id from` (select id from x) as a group by a.id\n\nThe 4.X version supports limited sub-queries. This problem is caused by the name of the intermediate table. Remove the table alias of `select` or `group order` or other fields.\n\n> [https://github.com/apache/shardingsphere/issues/4810](https://github.com/apache/shardingsphere/issues/4810)\n\n**7. The table’s primary key conflicts with the primary key generated by the [SNOWFLAKE](https://programming.vip/docs/overview-of-snowflake-algorithm.html) algorithm.**\n\nShardingSphere provides flexible ways to configure distributed primary key generation strategies. In the sharding rule configuration module, you can configure the primary key generation strategy for each table.\n\nBy default, the [snowflake](https://programming.vip/docs/overview-of-snowflake-algorithm.html) algorithm is used to generate long integer data of 64bit. The snowflake generator needs to be configured with:\n\n`spring.shardingsphere.sharding.tables.x.key-generator.props.worker.id = ${dcc.node.id}`\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/dbopmsazvtztkb0p2m0z.png)\n\n\n\nThe company uses the [apollo](https://www.apolloconfig.com/#/) configuration center to deliver the node id of the service instance. The service uses multi-data sources. If you use the YAML file to load sharding configuration, the `workId` cannot be automatically loaded into sharding configuration items.\n\n**Solutions:**\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/mfxy2xgsl6w0bwgi0hqv.png)\n\n\nUse the custom generator type based on the built-in `SnowflakeShardingKeyGenerator`.\n\nIf the primary key is used as a shard key, configure `max.vibration.offset` based on the data sharding value to increase the vibration range.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/7jo25gkyz57kyttllvsd.png)\n\n\n\n**8. The 3.X version reports an error when `CASE WHEN` statement is executed.**\n\nFirst, the 3.X and 4.X versions don’t support the `case when` statement.\n\nThe 3.X and 4.X versions have different logics when parsing the shard keys of `case when`'s `update` statement. The 4.X `parserEngine.parse` method will ignore the `case when` parsing parameters, resulting in inconsistency with the external parameter list and an error when 3.X executes the normal SQL.\n\nThe 3.X version works correctly because the first parameter of `case when` is intentionally set to the shard key when the SQL is written, and the `case when` statement comes first.\n\n> [https://github.com/apache/shardingsphere/issues/13233](https://github.com/apache/shardingsphere/issues/13233)\n\n**Solutions：**\n\n- It is suggested to rewrite SQL as the `case when` is not supported.\n- According to the shard key parsing logic in version 4.1.1, `case when` is placed at the end, and the shard key remains the first parameter of `case when`.\n\n**9. The logical table `actualDataNodes` is configured and no default value error is reported for the primary key.**\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/7tpr0fitx420itauw5f8.png)\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/pprbsvn5n8tdrm2mezs7.png)\n\n\n\n\nThe `check.table.metadata.enabled=true` is not configured for service, and the metadata consistency of sub-tables is not checked by default.\n\nThe first table of `actualDataNodes` configured by services does not exist, resulting in an empty `GenerateKeyContenxt`.\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/a75kpppwhniaubgw3iti.png)\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/cw16ejpxmwvcva9bd9t4.png)\n\n\n\n\n**Solutions：**\n\n- Configure `check.table.metadata.enabled=true`. A non-existent table is detected when started and an error is reported.\n- Rewrite the `actualDataNodes inline` expression to make sure that the first table exists.\n\n**10. In version 3.0, there is a deadlock under the high concurrency of the full database and table router.**\n\nShardingSphere-JDBC uses local transactions by default. In local transactions, the database connection is obtained asynchronously. Under high concurrency, it is possible that all database connections cannot be obtained, resulting in a deadlock.\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/5eo47dqy991nmvu048k8.png)\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/hdnnfskjri9un1wj6n3h.png)\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/y8fpd887zvkg5oovt4hm.png)\n\n\n\n![Image description](https://dev-to-uploads.s3.amazonaws.com/uploads/articles/pzqw4c89j1mplfmwfxz0.png)\n\n\n\n\n## Conclusion\n\nAs a [ShardingSphere](https://shardingsphere.apache.org/) core user, [Energy Monster](https://ir.enmonster.com/)’s upgrade process also reflects some problems that community users may encounter in the application of ShardingSphere.\n\nCurrently, Apache ShardingSphere’s stable version has been updated to 5.1.2 and has been optimized in terms of its functions, performance, testing, documentation, and examples.\n\nYou can refer to [Apache ShardingSphere’s official website](https://shardingsphere.apache.org/) for more information. If you have any questions or suggestions, you are also welcome to give feedback on [Github](https://github.com/apache/shardingsphere). The community will actively respond and discuss.\n\n**Project Links:**\n\n[ShardingSphere Github](https://github.com/apache/shardingsphere/issues?page=1&q=is%3Aopen+is%3Aissue+label%3A%22project%3A+OpenForce+2022%22)\n\n[ShardingSphere Twitter](https://twitter.com/ShardingSphere)\n\n[ShardingSphere Slack](https://join.slack.com/t/apacheshardingsphere/shared_invite/zt-sbdde7ie-SjDqo9~I4rYcR18bq0SYTg)\n\n[Contributor Guide](https://shardingsphere.apache.org/community/cn/contribute/)\n\n[GitHub Issues\n](https://github.com/apache/shardingsphere/issues)\n[Contributor Guide](https://shardingsphere.apache.org/community/en/contribute/)","date":"2022-09-06","author":"SphereEx","excerpt":"NASDAQ-listed Energy Monster is the largest provider of mobile device charging services in Asia. This article illustrates how Energy Monster uses ShardingSphere-JDBC.","metaDescription":"Energy Monster is a power bank rental company, allowing its users to rent power banks by scanning QR codes. It is designed to help people to live their life without interruptions. Energy Monster reportedly covers 100 cities and 100K merchants. ","createdAt":"2022-09-26T06:27:13.410Z","updatedAt":"2022-09-28T02:42:03.843Z","publishedAt":"2022-09-27T06:55:09.446Z","locale":"en","caseStudiesType":{"data":{"id":9,"attributes":{"name":"Internet","createdAt":"2022-09-26T10:34:03.086Z","updatedAt":"2022-09-27T06:59:34.838Z","publishedAt":"2022-09-27T06:59:34.836Z","locale":"en"}}},"cover":{"data":{"id":334,"attributes":{"name":"energy monster.png","alternativeText":"energy monster.png","caption":"energy monster.png","width":1093,"height":489,"formats":{"thumbnail":{"name":"thumbnail_energy monster.png","hash":"thumbnail_energy_monster_2f744c0ab8","ext":".png","mime":"image/png","width":245,"height":110,"size":18,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_energy_monster_2f744c0ab8.png"},"large":{"name":"large_energy monster.png","hash":"large_energy_monster_2f744c0ab8","ext":".png","mime":"image/png","width":1000,"height":447,"size":191.25,"path":null,"url":"https://official-media.sphere-ex.com/large_energy_monster_2f744c0ab8.png"},"medium":{"name":"medium_energy monster.png","hash":"medium_energy_monster_2f744c0ab8","ext":".png","mime":"image/png","width":750,"height":336,"size":115.92,"path":null,"url":"https://official-media.sphere-ex.com/medium_energy_monster_2f744c0ab8.png"},"small":{"name":"small_energy monster.png","hash":"small_energy_monster_2f744c0ab8","ext":".png","mime":"image/png","width":500,"height":224,"size":58.44,"path":null,"url":"https://official-media.sphere-ex.com/small_energy_monster_2f744c0ab8.png"}},"hash":"energy_monster_2f744c0ab8","ext":".png","mime":"image/png","size":182.88,"url":"https://official-media.sphere-ex.com/energy_monster_2f744c0ab8.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T06:34:45.604Z","updatedAt":"2022-09-26T06:34:45.604Z"}}},"metaLogo":{"data":{"id":349,"attributes":{"name":"Energy monster logo.png","alternativeText":"Energy monster logo.png","caption":"Energy monster logo.png","width":373,"height":298,"formats":{"thumbnail":{"name":"thumbnail_Energy monster logo.png","hash":"thumbnail_Energy_monster_logo_102b9745bc","ext":".png","mime":"image/png","width":195,"height":156,"size":8.8,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_Energy_monster_logo_102b9745bc.png"}},"hash":"Energy_monster_logo_102b9745bc","ext":".png","mime":"image/png","size":17.3,"url":"https://official-media.sphere-ex.com/Energy_monster_logo_102b9745bc.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:19:26.166Z","updatedAt":"2022-09-26T08:19:26.166Z"}}},"localizations":{"data":[]}}},{"id":25,"attributes":{"feature":true,"title":"Apache ShardingSphere & iQiyi: What's Database Mesh?","content":"> _To further understand application scenarios, enterprises’ needs, and improve dev teams’ understanding of Apache ShardingSphere, SphereEx launched the “Enterprise Visits” series. The second stop took us to the leading online streaming platform in Asia, iQiyi._\n\nIn November 2021, a team from the ShardingSphere community visited the innovation center of iQiyi for in-depth interactions and discussions with their counterparts from Beijing and Shanghai. During the meetup, Zhang Liang, Apache ShardingSphere PMC Chair and SphereEx Founder provided details on the latest Apache ShardingSphere community initiatives, its future development, and Database Mesh.\n\nDuring the meetup, iQiyi was especially interested in the capabilities and future plans of ShardingSphere and Database Mesh. here are some key takeaway questions from the visit:\n\n**Q: How does ShardingSphere manage its encryption keys?**\n\n[Encryption](https://shardingsphere.apache.org/document/current/en/features/encrypt/) key management is a capability subordinate to the encryption algorithm. When ShardingSphere needs an encryption key in its decryption, the key could be passed through properties configured by the encryption algorithm. In principle, encryption key management is not a capability of ShardingSphere but the job of the encryption algorithm.\n\n**Q:Is there a built-in cryptographic algorithm in ShardingSphere? Or is it done through outside cooperation?**\n\nStrictly speaking, ShardingSphere is not equipped with built-in capability. That’s because the pluggable architecture of ShardingSphere is divided into multiple layers, and the encryption module itself is only a capability of L2 of the pluggable architecture.\n\nThe encryption module only defines the top-layer encryption connector and the detailed encryption algorithm achieves the pluggable part of the connector. Currently, ShardingSphere supports several of the most commonly used open source algorithms, but it does not support any national cryptographic algorithm in particular.\n\nIntegrating an encryption algorithm is easy for developers: instead of changing source codes, all they need is a ShardingSphere encryption algorithm connector. ShardingSphere’s data encryption process is completely transparent to users and very easy to use.\n\n**What’s the future plan for Database Mesh?**\n\nDatabase Mesh is another ShardingSphere initiative. We are now planning to integrate Database Mesh through Sidecar. Sidecar will be used to manage traffic, while ShardingSphere will be used to manage to compute, and the underlying databases’ nodes to manage storage.\n\nSidecar’s and ShardingSphere functions overlap in part, but not completely. ShardingSphere has more than 190 modules, therefore it’s not possible to completely replace ShardingSphere with Sidecar. That’s why ShardingSphere empowers some lightweight capabilities to Sidecar, and Sidecar is equipped with the capabilities to buffer some traffic and requests.\n\nThe database computing capability is still processed by ShardingSphere, as it would consume too many application resources to run heavyweight computation via Sidecar for the distributed transaction, query and optimization of heterogeneous databases.\n\nIn the future, other Sidecar capabilities, such as management, SQL auditing and authorities, will be developed. In this way, Sidecar and ShardingSphere will have their own focuses, providing a better solution. \n\n**Sidecar is equivalent to a data plane. Will you combine the control plane with [Istio](https://istio.io/) or will you build the control plane separately?**\n\nShardingSphere uses DistSQL (Distributed SQL) to create rules for sharding, encryption and so on. ShardingSphere is already equipped with the control plane capability of Sidecar. Any capability achieved on Sidecar can be controlled by ShardingSphere’s [DistSQL](https://opensource.com/article/21/9/distsql). In the future, we will consider integrating Sidecar into the Istio ecosystem.\n\nSo far, Apache ShardingSphere with the release of a version of 5.0 has officially repositioned itself in a new field with the Database Plus concept at its core. In addition to the powerful incremental capability at the upper layer of databases, the Database Plus architecture also opens a highly extensible database ecosystem to developers and users, charting a new future development course for Apache ShardingSphere.\n\n**Apache ShardingSphere Open Source Project Links:**\n\n[ShardingSphere Github]\n(https://github.com/apache/shardingsphere/issues?page=1&q=is%3Aopen+is%3Aissue+label%3A%22project%3A+OpenForce+2022%22)\n\n[ShardingSphere Twitter](https://twitter.com/ShardingSphere)\n\n[Contributor Guideline](https://shardingsphere.apache.org/community/cn/contribute/)","date":"2022-01-10","author":"Yacine Si Tayeb","excerpt":"Also known as the Netflix of Asia, iQiyi is an ad-supported television and movie portal providing fully-licensed, high-definition media content.","metaDescription":null,"createdAt":"2022-09-26T07:52:25.745Z","updatedAt":"2022-09-28T02:38:53.315Z","publishedAt":"2022-09-27T07:00:31.240Z","locale":"en","caseStudiesType":{"data":{"id":9,"attributes":{"name":"Internet","createdAt":"2022-09-26T10:34:03.086Z","updatedAt":"2022-09-27T06:59:34.838Z","publishedAt":"2022-09-27T06:59:34.836Z","locale":"en"}}},"cover":{"data":{"id":389,"attributes":{"name":"iqiyi2.jpg","alternativeText":"iqiyi2.jpg","caption":"iqiyi2.jpg","width":500,"height":360,"formats":{"thumbnail":{"name":"thumbnail_iqiyi2.jpg","hash":"thumbnail_iqiyi2_3b19419f07","ext":".jpg","mime":"image/jpeg","width":217,"height":156,"size":5.22,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_iqiyi2_3b19419f07.jpg"}},"hash":"iqiyi2_3b19419f07","ext":".jpg","mime":"image/jpeg","size":12.26,"url":"https://official-media.sphere-ex.com/iqiyi2_3b19419f07.jpg","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-28T02:38:45.900Z","updatedAt":"2022-09-28T02:38:45.900Z"}}},"metaLogo":{"data":null},"localizations":{"data":[]}}},{"id":6,"attributes":{"feature":true,"title":"Apache ShardingSphere & JD Baitiao: Story of an Implementation Journey","content":">Apache ShardingSphere has become the best solution for JD Baitiao’s very large storage scenarios and scale-up. JD’s shopping festival weighed very much on our big data storage. I mean, like hundreds of millions of data sets.\n>\n>JD Baitiao, an internet credit products provider launched in 2014, has always been challenged by data explosion. Every shopping festival is like a technical test for us, but every time the technical staff professionals take proactive strategic initiatives in data architecture development to ensure everything goes smoothly.\n>\n>— Dongfang Zhang，JD Batiao R&D Director\n\n## The Baitiao Data Architecture Evolution\n\nLaunched in Feb. 2014, JD Baitiao’s data architecture has already evolved for several times because of rapid increase in data volume and rapid business growth.\n\n### From 2014 to 2015\n\nBaitiao first adopted the Solr + HBase solution to connect core business systems and other systems with key databases. Solr indexing is able to achieve fast search responses and HBase is used to store a massive amount of data.\n\n- A Solr cluster can help the core database reduce reads and writes stress.\n- However, Solr is not perfect: it has unsatisfying extensibility and intrusions.\n\n### From 2015 to 2016\n\nThe company chose NoSQL to store monthly data tables in Mongo DB clusters, temporarily meeting the demand for importing/exporting massive data in payment settlement scenarios.\n\n- The change made it quite efficient to query hot data. In addition, thanks to such unstructured data storage, staff can easily modify table structures.\n- Nevertheless, the plan was still problematic: bad extensibility, intrusions, and memory hog.\n\n### From 2016 to 2017\n\nGrowing business and growing data. Tens of billions of data volume imposed more performance and volume pressure on MongoDB. JD Baitiao’s Big Data Platform used DBRep and MySQL Slave to capture data changes and store the information data in its message center, to later be written to ES and HBase.\n\n- This plan was actually much better. It focused on real-time data and improved extensibility.\n- However, Baitiao faced high costs to maintain its code because of the data sharding problem in its architecture.\n\nThe architecture evolution of JD Baitiao is just an illustration of rapidly growing Internet consumer finance. All the methods Baitiao adopted in the past were not comprehensive, indicating that every solution would be outdated soon.\n\n## Time to Adopt a Decoupled Architecture\n\nTo ensure good system performance even with rapidly increasing data volumes, the technical team first chose the sharding architecture pattern that not only guaranteed excellent performance, but also kept code under control.\n\n### The data striping plan was actually based on application architecture:\n\nThe product continuously developed but the early solution had now become one of the biggest problems. The old data sharding plan makes code even more complex and increases maintenance expenditure. Developers have to spend much time adjusting sharding every time when the application upgrades, so they cannot concentrate on their own developments. In fact, tight coupling is to blame for the problem.\n\nTherefore, the team decided to use a mature sharding component that simplifies system upgrades and architecture changes. To compare Baitiao’s shard and ShardingSphere shard, their differences are shown in the following table.\n\n![jd_en_1.png](https://official-media.sphere-ex.com/jd_en_1_eb365b4a93.png)\n\nFor Baitiao, decoupling was the next task.\n\nApparently, JD Baitiao‘s data architecture will experience a brand-new journey to decoupling. The following three directions definitely fueled its transformation:\n\n**Concentration:** instead of database sharding built in its architecture, it applied a sharding component to place more energy on it own product development.\n\n**Easier Upgrade:** it used a decoupled architecture to simplify the R&D process of system upgrades.\n\n**Future-Oriented Plan:** it aims to improve system extensibility so that Baitiao will be more capable of confidently holding large online shopping festivals like the 618 Shopping Festival and Double 11 Shopping Festival (also known as Singles Day on November 11th).\n\nJD Baitiao is a huge business, so its business scenario is truly related to finance, high concurrency, and massive data volumes. Baitiao’s sharding component must have the following features:\n\n1. Be a mature product\n2. Have excellent performance\n3. Be able to handle big data\n4. Have an extensible architecture\n\n## The Apache ShardingSphere Solution\n\nThe lightweight Java framework ShardingSphere-JDBC is Apache ShardingSpheres’s first product providing services such as a Java Database Connectivity (JDBC) API. ShardingSphere JDBC uses jar archive file installation package and allows the client-side to directly connect to databases. So, it requires no extra deployment dependencies. It is like an enhanced JDBC driver fully compatible with JDBC and ORM frameworks.\n\nShardingSphere — JDBC provides the following features, making it the best solution in the Baitiao scenario.\n\nMature Product: ShardingSphere — JDBC is a mature product developed for years, and its open source community is very active.\n\nExcellent Performance: its micro-kernel and lightweight design hardly hinder performance.\n\nMinimum Changes: it supports native MySQL protocol and minimizes R&D workload.\n\nExtensibility: users can combine with the migration sync component to easily extend data.\n\n![jd_en_2.png](https://official-media.sphere-ex.com/jd_en_2_c92fb9e5ef.png)\n\nAfter being systematically tested for several times, Apache ShardingSphere became JD Baitiao’s first choice of data sharding middleware. The cooperation started at the end of 2018.\n\n## Product Adaptation\n\nTo provide Baitiao with better services and support the business, Apache ShardingSphere has made many improvements to its product features and performance during the implementation process. At the same time, the user case helps optimize the product in return.\n\n### Upgrade the SQL Engine\n\nBaitiao’s business logic is extremely complicated; its diversified scenario demands require the best SQL compatibility levels. Thus, Apache ShardingSphere restructures the SQL parser module to support more SQLs.\n\n- Single data node routing: 100% SQL compatibility\n- Multi data node routing：comprehensively support DML, DDL, DCL, TCL and partly DAL. Support features such as pagination, deduplication, ranking, grouping, aggregation and correlated query.\n\n### Distributed Key\n\nApache ShardingSphere provides built-in distributed key generators, such as UUID, and SNOWFLAKE. It also provides its distributed key generation API so that users can develop custom key generation algorithms to meet their special needs.\n\n### Shard Key Value Injection\n\nIf a SQL has no shard condition, Apache ShardingSphere uses ThreadLoad to manage shard key value, and users can program and add a shard condition to HintManager, making the condition only effective in the current thread. That’s so-called SQL zero intrusion.\n\nAdditionally, Apache ShardingSphere continues to optimize its other features in order to meet Baitiao’s need for high performance such as：\n\n- SQL parsing result cache\n- JDBC metadata cache\n- Bind table & Broadcast table\n- Automated execution engine & Stream merge\n\nJD Baitiao & ShardingSphere teams joint efforts amde all indicators of the product live up to their expectation. The resulting final performance is almost the same as that of a native JDBC.\n\n![jd_en_3.png](https://official-media.sphere-ex.com/jd_en_3_582e2afe89.png)\n\n## Cutover\n\nApache ShardingSphere uses its custom HASH strategy to shard data, effectively avoiding the hot data problem. The total of data nodes almost reaches ten thousand. The cutover process lasts for about 4 weeks.\n\n1. After DBRep reads data, Apache ShardingSphere synchronizes the data to the target database cluster.\n2. Two clusters run together. After a data migration, Baitiao uses its own tool to verify business and data.\n\nDBRep is fundamental to the product design of ShardingSphere-Scaling. The automation feature of Scaling definitely facilitates migration and scale-up.\n\n## Apache ShardingSphere Brings Many Benefits\n\n### Simplified Upgrade Path\n\nThe decoupled architecture effectively simplifies a technology stack required in a system upgrade, so the developer team no longer need to worry about table sharding. Instead, they can focus on the business itself. In brief, ShardingSphere greatly helps Baitiao optimize the upgrade path.\n\n### Save R&D Cost\n\nBaitiao directly uses the mature product Apache ShardingSphere, so they do not need to reinvent the wheel and save a lot of time and energy.\n\n### Improve Architecture Extensibility\n\nBaitiao also uses the sync migration component Scaling that makes system scale-up even more flexible. It’s the company’s secret for online shopping festival successes.\n\n## Summary\n\nThe growth of JD Baitiao’s business greatly stimulates its data architecture upgrades. This time, it chose Apache ShardingSphere to decouple its architecture and make future upgrades less complicated. Thank to this from now on, developers only have to concentrate on their own product optimization because its data architecture is extensible enough. The case is really a good example of ShardingSphere application in the consumer finance scenario.\n\nNow, there are more and more internet credit payment models. In the future, Apache ShardingSphere in collaboration with JD Technology will explore more business scenarios, and contribute finance-related technological innovations, further improving Internet finance.\n\n### About Apache ShardingSphere\n\nShardingSphere is one of the Apache Software Foundation Top-Level Open Source projects, used by over 170 enterprises worldwide, across various verticals such as finance, e-commerce, cloud services, tourism, logistics, education, and entertainment. Its GitHub community has so far cumulated over 14,000 stars.\n\nWe welcome more technical professionals to write articles and share their experiences and thoughts. If you are interested, feel free to contact us:\n\nShardingSphere Github: https://github.com/apache/shardingsphere\n\nShardingSphere Twitter: https://twitter.com/ShardingSphere\n\nShardingSphere Slack Channel: apacheshardingsphere.slack.com","date":"2021-12-01","author":"SphereEx","excerpt":"Apache ShardingSphere has become the best solution for JD Baitiao’s very large storage scenarios and scale-up. JD’s shopping festival weighed very much on our big data storage. I mean, like hundreds of millions of data sets.","metaDescription":"JD Technology Group is a business subgroup of JD Group that focuses on technology as an industry service, and is committed to providing technical products and solutions for enterprises, financial institutions, governments and other types of customers across the value chain.","createdAt":"2021-12-12T12:30:44.957Z","updatedAt":"2022-09-26T09:16:15.245Z","publishedAt":"2021-12-12T12:35:08.683Z","locale":"en","caseStudiesType":{"data":{"id":2,"attributes":{"name":"E-commerce","createdAt":"2021-12-11T23:02:08.395Z","updatedAt":"2021-12-11T23:02:11.020Z","publishedAt":"2021-12-11T23:02:10.947Z","locale":"en"}}},"cover":{"data":{"id":345,"attributes":{"name":"JDT.png","alternativeText":"JDT.png","caption":"JDT.png","width":720,"height":540,"formats":{"thumbnail":{"name":"thumbnail_JDT.png","hash":"thumbnail_JDT_2704eac855","ext":".png","mime":"image/png","width":208,"height":156,"size":8.85,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_JDT_2704eac855.png"},"small":{"name":"small_JDT.png","hash":"small_JDT_2704eac855","ext":".png","mime":"image/png","width":500,"height":375,"size":28.29,"path":null,"url":"https://official-media.sphere-ex.com/small_JDT_2704eac855.png"}},"hash":"JDT_2704eac855","ext":".png","mime":"image/png","size":14.08,"url":"https://official-media.sphere-ex.com/JDT_2704eac855.png","previewUrl":null,"provider":"strapi-provider-upload-s3-compat","provider_metadata":null,"createdAt":"2022-09-26T08:02:50.084Z","updatedAt":"2022-09-26T08:02:50.084Z"}}},"metaLogo":{"data":{"id":10,"attributes":{"name":"jd_logo.png","alternativeText":"jd_logo.png","caption":"jd_logo.png","width":300,"height":44,"formats":{"thumbnail":{"name":"thumbnail_jd_logo.png","hash":"thumbnail_jd_logo_591b2e5e14","ext":".png","mime":"image/png","width":245,"height":36,"size":6.43,"path":null,"url":"https://official-media.sphere-ex.com/thumbnail_jd_logo_591b2e5e14.png"}},"hash":"jd_logo_591b2e5e14","ext":".png","mime":"image/png","size":2.36,"url":"https://official-media.sphere-ex.com/jd_logo_591b2e5e14.png","previewUrl":null,"provider":"dine-strapi-provider-upload-aws-s3","provider_metadata":null,"createdAt":"2021-12-12T12:04:06.233Z","updatedAt":"2021-12-14T07:19:09.532Z"}}},"localizations":{"data":[{"id":1,"attributes":{"feature":true,"title":"Apache ShardingSphere 在京东白条场景的落地之旅","content":"\n>京东白条使用 Apache ShardingSphere 解决了千亿数据存储和扩容的问题，为大促活动奠定了基础。\n>\n>2014 年初，“京东白条”作为业内互联网信用支付产品，数据量爆发式的增长，每一次大促备战都是对技术人员的考验，每一次的战略转型驱动着数据架构的成长。\n>\n>— 张栋芳，京东白条研发负责人\n\n### 项目背景\n\n自 2014 年 2 月京东白条业务上线起，为满足快速发展的业务和激增的海量数据，白条的数据架构经历了数次演进。\n\n白条数据架构的演进间接地反应了互联网消费金融的飞速发展，也说明了每一种解决方案在不同背景下都有不同的保质期。\n\n### 业务挑战\n\n随着产品升级迭代，早期的解决方案演变成为了眼前的问题，通过业务框架实现的数据分片方案导致业务代码复杂度增加、维护成本不断攀升，紧耦合的弊端原形毕露，应用每次升级都需要投入较多的精力对分片做相应调整，研发人员难以专注于业务本身。\n\n技术团队经权衡后开始考虑使用成熟的分库分表组件来承担这部分工作，让业务系统升级和架构调整不再复杂。\n\n### 解决方案\n\nShardingSphere-JDBC 是 Apache ShardingSphere 的第一款产品，它定位为轻量级 Java 框架，在 Java 的 JDBC 层提供的额外服务。它使用客户端直连数据库，以 jar 包形式提供服务，无需额外部署和依赖，可理解为增强版的 JDBC 驱动，完全兼容 JDBC 和各种 ORM 框架。\n\nShardingSphere-JDBC 的以下特点能够很好地满足白条业务场景：\n\n- 产品成熟：经数年打磨产品成熟度高，且社区活跃；\n- 性能良好：微内核、轻量化的设计，性能损耗极小；\n- 改造量小：支持原生的 MySQL 协议，研发工作量小；\n- 扩展灵活：搭配使用迁移同步组件轻松实现数据扩展。\n\n经内部大量系统性验证之后，Apache ShardingSphere 成为了京东白条数据分片中间件的首选方案，2018 年底正式开始对接。\n\n![jd_zh_1.png](https://official-media.sphere-ex.com/jd_zh_1_fac28cd6ec.png)\n\n### 用户收益\n\n- 简化升级路径\n\n通过架构解耦，业务系统升级所涉及技术栈得到有效缩短，研发团队不再需要关注分表设计，精力全部聚焦于业务本身，升级路径得到大幅度优化；\n\n- 节省研发力量\n\n引入成熟的 Apache ShardingSphere 无需重新开发分表组件，在简化业务升级路径的基础上节省了大量研发力量；\n\n- 架构灵活扩展\n\n搭配使用 Scaling 同步迁移组件从容面对“618”和“11.11”等大型活动，系统灵活扩容。\n\n**SphereEx，让数据与应用连接更简单。**\n\n","date":"2021-12-01","author":"SphereEx","excerpt":"京东白条使用 Apache ShardingSphere 解决了千亿数据存储和扩容的问题，为大促活动奠定了基础。","metaDescription":"京东科技集团是京东集团旗下专注于以技术为产业服务的业务子集团，致力于为企业、金融机构、政府等各类客户提供全价值链的技术性产品与解决方案。","createdAt":"2021-12-11T23:10:03.172Z","updatedAt":"2021-12-14T07:22:15.304Z","publishedAt":"2021-12-11T23:10:07.307Z","locale":"zh"}}]}}}]}}}},
    "staticQueryHashes": []}