1/* 2MMJ's Cel Shader - v1.03 3---------------------------------------------------------------- 4-- 180403 -- 5This is a port of my old shader from around 2006 for Pete's OGL2 6plugin for ePSXe. It started out as a shader based on the 7"CComic" shader by Maruke. I liked his concept, but I was 8looking for something a little different in the output. 9 10Since the last release, I've seen some test screenshots from MAME 11using a port of my original shader and have also seen another 12port to get it working with the PCSX2 emulator. Having recently 13seen some Kingdom Hearts II and Soul Calibur 3 YouTube videos with 14my ported shader inspired me to revisit it and get it working in 15RetroArch. 16 17As for this version (1.03), I've made a few small modifications 18(such as to remove the OGL2Param references, which were specific 19to Pete's plugin) and I added some RetroArch Parameter support, 20so some values can now be changed in real time. 21 22Keep in mind, that this was originally developed for PS1, using 23various 3D games as a test. In general, it will look better in 24games with less detailed textures, as "busy" textures will lead 25to more outlining / messy appearance. Increasing "Outline 26Brightness" can help mitigate this some by lessening the 27"strength" of the outlines. 28 29Also (in regards to PS1 - I haven't really tested other systems 30too much yet), 1x internal resolution will look terrible. 2x 31will also probably be fairly blurry/messy-looking. For best 32results, you should probably stick to 4x or higher internal 33resolution with this shader. 34 35Parameters: 36----------- 37White Level Cutoff = Anything above this luminance value will be 38 forced to pure white. 39 40Black Level Cutoff = Anything below this luminance value will be 41 forced to pure black. 42 43Shading Levels = Determines how many color "slices" there should 44 be (not counting black/white cutoffs, which are always 45 applied). 46 47Saturation Modifier = Increase or decrease color saturation. 48 Default value boosts saturation a little for a more 49 cartoonish look. Set to 0.00 for grayscale. 50 51Outline Brightness = Adjusts darkness of the outlines. At a 52 setting of 1, outlines should be disabled. 53 54Shader Strength = Adjusts the weight of the color banding 55 portion of the shader from 0% (0.00) to 100% (1.00). At a 56 setting of 0.00, you can turn off the color banding effect 57 altogether, but still keep outlines enabled. 58----------- 59MMJuno 60*/ 61 62// Parameter lines go here: 63#pragma parameter WhtCutoff "White Level Cutoff" 0.97 0.50 1.00 0.01 64#pragma parameter BlkCutoff "Black Level Cutoff" 0.03 0.00 0.50 0.01 65#pragma parameter ShdLevels "Shading Levels" 16.0 1.0 16.0 1.0 66#pragma parameter SatModify "Saturation Modifier" 1.32 0.00 2.00 0.01 67#pragma parameter OtlModify "Outline Brightness" 0.20 0.00 1.00 0.01 68#pragma parameter ShdWeight "Shader Strength" 0.50 0.00 1.00 0.01 69 70 71#if defined(VERTEX) 72 73#if __VERSION__ >= 130 74#define COMPAT_VARYING out 75#define COMPAT_ATTRIBUTE in 76#define COMPAT_TEXTURE texture 77#else 78#define COMPAT_VARYING varying 79#define COMPAT_ATTRIBUTE attribute 80#define COMPAT_TEXTURE texture2D 81#endif 82 83#ifdef GL_ES 84#define COMPAT_PRECISION mediump 85#else 86#define COMPAT_PRECISION 87#endif 88 89COMPAT_ATTRIBUTE vec4 VertexCoord; 90COMPAT_ATTRIBUTE vec4 COLOR; 91COMPAT_ATTRIBUTE vec4 TexCoord; 92COMPAT_VARYING vec4 COL0; 93COMPAT_VARYING vec4 TEX0; 94COMPAT_VARYING vec4 TEX1; 95COMPAT_VARYING vec4 TEX2; 96COMPAT_VARYING vec4 TEX3; 97 98vec4 _oPosition1; 99uniform mat4 MVPMatrix; 100uniform COMPAT_PRECISION int FrameDirection; 101uniform COMPAT_PRECISION int FrameCount; 102uniform COMPAT_PRECISION vec2 OutputSize; 103uniform COMPAT_PRECISION vec2 TextureSize; 104uniform COMPAT_PRECISION vec2 InputSize; 105 106// compatibility #defines 107#define vTexCoord TEX0.xy 108#define SourceSize vec4(TextureSize, 1.0 / TextureSize) //either TextureSize or InputSize 109#define OutSize vec4(OutputSize, 1.0 / OutputSize) 110 111void main() 112{ 113 vec4 offset; 114 115 gl_Position = MVPMatrix * VertexCoord; 116 117 TEX0 = TexCoord.xyxy; 118 119 offset.xy = -(offset.zw = vec2(SourceSize.z, 0.0)); 120 TEX1 = TEX0 + offset; 121 122 offset.xy = -(offset.zw = vec2(0.0, SourceSize.w)); 123 TEX2 = TEX0 + offset; 124 TEX3 = TEX1 + offset; 125} 126 127#elif defined(FRAGMENT) 128 129#if __VERSION__ >= 130 130#define COMPAT_VARYING in 131#define COMPAT_TEXTURE texture 132out vec4 FragColor; 133#else 134#define COMPAT_VARYING varying 135#define FragColor gl_FragColor 136#define COMPAT_TEXTURE texture2D 137#endif 138 139#ifdef GL_ES 140#ifdef GL_FRAGMENT_PRECISION_HIGH 141precision highp float; 142#else 143precision mediump float; 144#endif 145#define COMPAT_PRECISION mediump 146#else 147#define COMPAT_PRECISION 148#endif 149 150uniform COMPAT_PRECISION int FrameDirection; 151uniform COMPAT_PRECISION int FrameCount; 152uniform COMPAT_PRECISION vec2 OutputSize; 153uniform COMPAT_PRECISION vec2 TextureSize; 154uniform COMPAT_PRECISION vec2 InputSize; 155uniform sampler2D Texture; 156COMPAT_VARYING vec4 TEX0; 157COMPAT_VARYING vec4 TEX1; 158COMPAT_VARYING vec4 TEX2; 159COMPAT_VARYING vec4 TEX3; 160 161// compatibility #defines 162#define Source Texture 163#define vTexCoord (TEX0.xy * TextureSize.xy / InputSize.xy) 164 165#define SourceSize vec4(TextureSize, 1.0 / TextureSize) //either TextureSize or InputSize 166#define OutSize vec4(OutputSize, 1.0 / OutputSize) 167 168#ifdef PARAMETER_UNIFORM 169uniform COMPAT_PRECISION float WhtCutoff; 170uniform COMPAT_PRECISION float BlkCutoff; 171uniform COMPAT_PRECISION float ShdLevels; 172uniform COMPAT_PRECISION float SatModify; 173uniform COMPAT_PRECISION float OtlModify; 174uniform COMPAT_PRECISION float ShdWeight; 175#else 176#define WhtCutoff 0.97 177#define BlkCutoff 0.03 178#define ShdLevels 9.0 179#define SatModify 1.15 180#define OtlModify 0.20 181#define ShdWeight 0.50 182#endif 183 184vec3 RGB2HSL(vec3 cRGB) 185{ 186 float cR = cRGB[0], cG = cRGB[1], cB = cRGB[2]; 187 float vMin = min(min(cR, cG), cB), vMax = max(max(cR, cG), cB); 188 float dMax = vMax - vMin, vS = 0.0, vH = 0.0, vL = (vMax + vMin) / 2.0; 189 190 // gray, no chroma 191 if(dMax == 0.0) { 192 vH = 0.0; vS = vH; 193 194 // chromatic data 195 } else { 196 if(vL < 0.5) { vS = dMax / (vMax + vMin); } 197 else { vS = dMax / (2.0 - vMax - vMin); } 198 199 float dR = (((vMax - cR) * 0.1667) + (dMax * 0.5)) / dMax; 200 float dG = (((vMax - cG) * 0.1667) + (dMax * 0.5)) / dMax; 201 float dB = (((vMax - cB) * 0.1667) + (dMax * 0.5)) / dMax; 202 203 if (cR >= vMax) { vH = dB - dG; } 204 else if(cG >= vMax) { vH = 0.3333 + dR - dB; } 205 else if(cB >= vMax) { vH = 0.6667 + dG - dR; } 206 207 if (vH < 0.0) { vH += 1.0; } 208 else if(vH > 1.0) { vH -= 1.0; } 209 } 210 return vec3(vH, vS, vL); 211} 212 213float Hue2RGB(float v1, float v2, float vH) 214{ 215 float v3 = 0.0; 216 217 if (vH < 0.0) { vH += 1.0; } 218 else if(vH > 1.0) { vH -= 1.0; } 219 220 if ((6.0 * vH) < 1.0) { v3 = v1 + (v2 - v1) * 6.0 * vH; } 221 else if((2.0 * vH) < 1.0) { v3 = v2; } 222 else if((3.0 * vH) < 2.0) { v3 = v1 + (v2 - v1) * (0.6667 - vH) * 6.0; } 223 else { v3 = v1; } 224 225 return v3; 226} 227 228vec3 HSL2RGB(vec3 vHSL) 229{ 230 float cR = 0.0, cG = cR, cB = cR; 231 232 if(vHSL[1] == 0.0) { 233 cR = vHSL[2], cG = cR, cB = cR; 234 235 } else { 236 float v1 = 0.0, v2 = v1; 237 238 if(vHSL[2] < 0.5) { v2 = vHSL[2] * (1.0 + vHSL[1] ); } 239 else { v2 = (vHSL[2] + vHSL[1] ) - (vHSL[1] * vHSL[2] ); } 240 241 v1 = 2.0 * vHSL[2] - v2; 242 243 cR = Hue2RGB(v1, v2, vHSL[0] + 0.3333); 244 cG = Hue2RGB(v1, v2, vHSL[0] ); 245 cB = Hue2RGB(v1, v2, vHSL[0] - 0.3333); 246 } 247 return vec3(cR, cG, cB); 248} 249 250vec3 colorAdjust(vec3 cRGB) 251{ 252 vec3 cHSL = RGB2HSL(cRGB); 253 254 float cr = 1.0 / ShdLevels; 255 256 // brightness modifier 257 float BrtModify = mod(cHSL[2], cr); 258 259 if (cHSL[2] > WhtCutoff) { cHSL[1] = 1.0; cHSL[2] = 1.0; } 260 else if(cHSL[2] > BlkCutoff) { cHSL[1] *= SatModify; cHSL[2] += (cHSL[2] * cr - BrtModify); } 261 else { cHSL[1] = 0.0; cHSL[2] = 0.0; } 262 cRGB = 1.2 * HSL2RGB(cHSL); 263 264 return cRGB; 265} 266 267 268void main() 269{ 270 vec3 c0 = COMPAT_TEXTURE(Source, TEX3.xy).rgb; 271 vec3 c1 = COMPAT_TEXTURE(Source, TEX2.xy).rgb; 272 vec3 c2 = COMPAT_TEXTURE(Source, TEX3.zy).rgb; 273 vec3 c3 = COMPAT_TEXTURE(Source, TEX1.xy).rgb; 274 vec3 c4 = COMPAT_TEXTURE(Source, TEX0.xy).rgb; 275 vec3 c5 = COMPAT_TEXTURE(Source, TEX1.zw).rgb; 276 vec3 c6 = COMPAT_TEXTURE(Source, TEX3.xw).rgb; 277 vec3 c7 = COMPAT_TEXTURE(Source, TEX2.zw).rgb; 278 vec3 c8 = COMPAT_TEXTURE(Source, TEX3.zw).rgb; 279 280 vec3 c9 = ((c0 + c2 + c6 + c8) * 0.15 + (c1 + c3 + c5 + c7) * 0.25 + c4) / 2.6; 281 282 vec3 o = vec3(1.0); vec3 h = vec3(0.05); vec3 hz = h; float k = 0.005; 283 float kz = 0.007; float i = 0.0; 284 285 vec3 cz = (c9 + h) / (dot(o, c9) + k); 286 287 hz = (cz - ((c0 + h) / (dot(o, c0) + k))); i = kz / (dot(hz, hz) + kz); 288 hz = (cz - ((c1 + h) / (dot(o, c1) + k))); i += kz / (dot(hz, hz) + kz); 289 hz = (cz - ((c2 + h) / (dot(o, c2) + k))); i += kz / (dot(hz, hz) + kz); 290 hz = (cz - ((c3 + h) / (dot(o, c3) + k))); i += kz / (dot(hz, hz) + kz); 291 hz = (cz - ((c5 + h) / (dot(o, c5) + k))); i += kz / (dot(hz, hz) + kz); 292 hz = (cz - ((c6 + h) / (dot(o, c6) + k))); i += kz / (dot(hz, hz) + kz); 293 hz = (cz - ((c7 + h) / (dot(o, c7) + k))); i += kz / (dot(hz, hz) + kz); 294 hz = (cz - ((c8 + h) / (dot(o, c8) + k))); i += kz / (dot(hz, hz) + kz); 295 296 i /= 8.0; i = pow(i, 0.75); 297 298 if(i < OtlModify) { i = OtlModify; } 299 c9 = min(o, min(c9, c9 + dot(o, c9))); 300 FragColor.rgb = mix(c4 * i, colorAdjust(c9 * i), ShdWeight); 301} 302#endif 303